{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([7000, 500]) torch.Size([7000, 1]) torch.Size([3000, 500]) torch.Size([3000, 1])\n",
      "torch.Size([256, 500])\n",
      "torch.Size([256])\n",
      "torch.Size([1, 256])\n",
      "torch.Size([1])\n",
      "epoch 1,train_loss 0.015323,test_loss 0.014876\n",
      "epoch 2,train_loss 0.014219,test_loss 0.013920\n",
      "epoch 3,train_loss 0.013122,test_loss 0.012973\n",
      "epoch 4,train_loss 0.011951,test_loss 0.011959\n",
      "epoch 5,train_loss 0.010661,test_loss 0.010823\n",
      "epoch 6,train_loss 0.009283,test_loss 0.009612\n",
      "epoch 7,train_loss 0.007856,test_loss 0.008333\n",
      "epoch 8,train_loss 0.006466,test_loss 0.007070\n",
      "epoch 9,train_loss 0.005196,test_loss 0.005899\n",
      "epoch 10,train_loss 0.004102,test_loss 0.004848\n",
      "epoch 11,train_loss 0.003223,test_loss 0.003983\n",
      "epoch 12,train_loss 0.002556,test_loss 0.003301\n",
      "epoch 13,train_loss 0.002059,test_loss 0.002771\n",
      "epoch 14,train_loss 0.001703,test_loss 0.002377\n",
      "epoch 15,train_loss 0.001449,test_loss 0.002088\n",
      "epoch 16,train_loss 0.001262,test_loss 0.001867\n",
      "epoch 17,train_loss 0.001124,test_loss 0.001703\n",
      "epoch 18,train_loss 0.001023,test_loss 0.001581\n",
      "epoch 19,train_loss 0.000945,test_loss 0.001488\n",
      "epoch 20,train_loss 0.000883,test_loss 0.001415\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEKCAYAAAAFJbKyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3gVZdrH8e+dTkIKhA6hg/QamlgXVwEVEAEBaQKiu7qr6+oqu+v6rltsu7YVRaQjCoqyoqBYwMIKgYBIRwJSQu8kkJ77/WMGCOGkQU5OEu7PdZ0rc2aemXOfk/LLPDPzjKgqxhhjTGH5+boAY4wxZYsFhzHGmCKx4DDGGFMkFhzGGGOKxILDGGNMkVhwGGOMKRKvBoeI9BSRrSKSICJPeFgeLCJz3eVxIlLfnR8tIktFJFlEXsu1TpCITBKRn0Rki4jc6c33YIwx5kIB3tqwiPgDE4BfAonAKhFZoKqbcjQbAxxX1cYiMhh4DrgLSAWeBFq5j5z+BBxS1aYi4gdU9tZ7MMYYczFv7nF0BhJUdYeqpgNzgL652vQFZrjT84AeIiKqelpVl+EESG6jgWcAVDVbVY94p3xjjDGeeG2PA6gN7MnxPBHoklcbVc0UkZNANOAxDEQkyp38m4jcAGwHHlTVg/kVUqVKFa1fv35R6zfGmCva6tWrj6hq1dzzvRkc4mFe7vFNCtMmpwCgDvA/VX1ERB4B/gUMv+jFRcYB4wDq1q1LfHx8oYo2xhjjEJFdnuZ7s6sqEYjJ8bwOsC+vNiISAEQCx/LZ5lHgDDDfff4+0MFTQ1WdpKqxqhpbtepFgWmMMeYSeTM4VgFNRKSBiAQBg4EFudosAEa60wOAJZrPqIvuso+BG9xZPYBNebU3xhhT/LzWVeUes3gQWAz4A1NVdaOIPA3Eq+oCYAowS0QScPY0Bp9dX0R2AhFAkIj0A252z8h63F3nZeAwcI+33oMxxpiLyZUwrHpsbKzaMQ5jypeMjAwSExNJTfV08qUpipCQEOrUqUNgYOAF80VktarG5m7vzYPjxhjjNYmJiYSHh1O/fn1EPJ1nYwpDVTl69CiJiYk0aNCgUOvYkCPGmDIpNTWV6OhoC43LJCJER0cXac/NgsMYU2ZZaBSPon6OFhz5mB23i++2HfZ1GcYYU6pYcOQhPTOb2St2M2Z6PIs3HvB1OcaYUubEiRO8/vrrRV6vd+/enDhxosjrjRo1innz5hV5PW+w4MhDUIAf797blZa1I/j17DXM/yHR1yUZY0qRvIIjKysr3/UWLVpEVFRUvm1KOwuOfESGBvL2mC50aVCZ3839kVkrPF59b4y5Aj3xxBNs376ddu3a0alTJ2688UaGDh1K69atAejXrx8dO3akZcuWTJo06dx69evX58iRI+zcuZPmzZtz77330rJlS26++WZSUlIK9dpfffUV7du3p3Xr1owePZq0tLRzNbVo0YI2bdrw6KOPAvD+++/TqlUr2rZty3XXXVcs791Oxy1AWHAAU0d14sF31vDkfzeQlJrBr29o7OuyjDE5/PXjjWzad6pYt9miVgRP3d4yz+XPPvssGzZsYO3atXz99dfceuutbNiw4dwprVOnTqVy5cqkpKTQqVMn7rzzTqKjoy/YxrZt23j33Xd56623GDRoEB988AHDhg3Lt67U1FRGjRrFV199RdOmTRkxYgRvvPEGI0aMYP78+WzZsgUROdcd9vTTT7N48WJq1659SV1kntgeRyGEBPrzxrCO9Glbi+c/28rzn23hSrhw0hhTeJ07d77gOohXX32Vtm3b0rVrV/bs2cO2bdsuWqdBgwa0a9cOgI4dO7Jz584CX2fr1q00aNCApk2bAjBy5Ei+/fZbIiIiCAkJYezYsXz44YeEhoYC0L17d0aNGsVbb71VYDdaYdkeR362LIRK9aF6SwL9/XjprnaEBQfw+tfbSU7L5P9ub4mfn50OaIyv5bdnUFLCwsLOTX/99dd8+eWXLF++nNDQUG644QaP10kEBwefm/b39y9UV1Ve/7QGBASwcuVKvvrqK+bMmcNrr73GkiVLmDhxInFxcSxcuJB27dqxdu3ai/Z8isqCIy9ZGbD4T5B8CPpPgua34e8n/POOVoSHBDDp2x0kp2by/IA2BPjbjpsxV5rw8HCSkpI8Ljt58iSVKlUiNDSULVu2sGLFimJ73WbNmrFz504SEhJo3Lgxs2bN4vrrryc5OZkzZ87Qu3dvunbtSuPGTpf69u3b6dKlC126dOHjjz9mz549Fhxe4x8I93wKc+92Hjf8Ea57DPHzY3yvZoQHB/DvL37idHomrw5pT3CAv68rNsaUoOjoaLp3706rVq2oUKEC1atXP7esZ8+eTJw4kTZt2nDVVVfRtWvXYnvdkJAQpk2bxsCBA8nMzKRTp07cf//9HDt2jL59+5Kamoqq8tJLLwHw2GOPsW3bNlSVHj160LZt28uuwQY5LEhGKnz8EKybA837QL83ILgiANP+9zN//XgT1zapwpvDOxIaZDlsTEnZvHkzzZs393UZ5YanzzOvQQ6tj6UggSFwx0S4+R+w5ROYegscd07Lvad7A54f0Ib/JRxhxJSVnEzJ8HGxxhjjfRYchSECVz8Id78PJ/bAWzfCzmUADIqN4T9DOvBj4gmGvrWCo8lpPi7WGFOWPfDAA7Rr1+6Cx7Rp03xd1gWsb6UoGt8E9y6BdwfDzL7Q6znoNJZb29QkNNif+2etZtCby3l7bBdqRlbwdbXGmDJowoQJvi6hQLbHUVRVGsO9X0GjX8DC38PHD0NmOjdeVY2Zoztz8FQaAycuZ9fR076u1BhjvMKC41KERMKQOdD9YVg9zdn7OH2ELg2jeefeLiSnZTJw4nJ+Ouj5VD1jjCnLLDgulZ8//PKv0H8y7FsDk26A/etoUyeKueO6ATDozeX8uKd4LvE3xpjSwqvBISI9RWSriCSIyBMelgeLyFx3eZyI1HfnR4vIUhFJFpHX8tj2AhHZ4M36C6XNQBj9GWi2c8bVxvlcVSOc9+/vRsXgAO6eHMeKHUd9XaUxxhQbrwWHiPgDE4BeQAtgiIi0yNVsDHBcVRsDLwHPufNTgSeBR/PYdn8g2Rt1X5Ja7eHepVCjNbw/Cpb8g3qVKvD+/d2oHhHMyKkrWbLloK+rNMYUo0u9HwfAyy+/zJkzZ/Jtc3YU3dLIm3scnYEEVd2hqunAHKBvrjZ9gRnu9Dygh4iIqp5W1WU4AXIBEakIPAL83XulX4Lw6jDyY2g/DL59HuYOo2ZIJu/d140m1SsybuZqFvy4z9dVGmOKibeDozTzZnDUBvbkeJ7ozvPYRlUzgZNAQYOo/A34N5Dvpy4i40QkXkTiDx8uodu/BgRDn9eg1/Pw02cw+ZdEp+/lnXu70qFuJR6a8wPvxO0umVqMMV6V834cjz32GC+88AKdOnWiTZs2PPXUUwCcPn2aW2+9lbZt29KqVSvmzp3Lq6++yr59+7jxxhu58cYbC/VaL774Iq1ataJVq1a8/PLLeW77bF2578lR3Lx5HYenYWNzj29SmDbnG4u0Axqr6u/OHg/Ji6pOAiaBM+RIvpUWJxHoch9UvQreGwmTbiRi4HRmjL6OX81ezR/nrycpNYP7rm9UYiUZU+59+gQcWF+826zRGno9m+finPfj+Pzzz5k3bx4rV65EVenTpw/ffvsthw8fplatWixcuBBwBj+MjIzkxRdfZOnSpVSpUqXAMlavXs20adOIi4tDVenSpQvXX389O3bsuGjbx44d83hPjuLmzT2ORCAmx/M6QO6+mnNtRCQAiASO5bPNbkBHEdkJLAOaisjXxVRv8Wp4A4xbChG14O3+VFg9kUnDOnJbm5o88+kWXlhs9/Qwprz4/PPP+fzzz2nfvj0dOnRgy5YtbNu2jdatW/Pll1/y+OOP89133xEZGVnkbS9btow77riDsLAwKlasSP/+/fnuu+88bjuve3IUN2/ucawCmohIA2AvMBgYmqvNAmAksBwYACzRfP6aquobwBsA7h7HJ6p6Q3EXXmwqN4QxX8B/74fFfyRo/zpeufMlwkMCmLB0O6dSMvlrH7unhzGXLZ89g5KgqowfP5777rvvomWrV69m0aJFjB8/nptvvpm//OUvRd62J02bNvW4bU/35Ch2quq1B9Ab+AnYDvzJnfc00MedDgHeBxKAlUDDHOvuxNn7SMbZM2mRa9v1gQ2FqaNjx47qU1lZql8/p/pUhOqb12v2iT36j4WbtN7jn+jDc37QjMws39ZnTBm0adMmn77+kSNHtG7duqqqunjxYu3cubMmJSWpqmpiYqIePHhQ9+7dqykpKaqqOn/+fO3bt6+qqrZq1Up37NiR7/br1aunhw8f1tWrV2vr1q319OnTmpycrC1bttQ1a9Z43HZSUpIePHhQVVWPHj2qlSpVKvT78fR5AvHq4W+qV8eqUtVFwKJc8/6SYzoVGJjHuvUL2PZOoNVlF1kS/Pzg+j9A9Vbw4Thk0o2MHzSTyApX8cLirSSnZfKfIe0JCbR7ehhTVuS8H0evXr0YOnQo3bo5F/9WrFiRt99+m4SEBB577DH8/PwIDAzkjTfeAGDcuHH06tWLmjVrsnTp0nxfp0OHDowaNYrOnTsDMHbsWNq3b8/ixYsv2nZSUpLHe3IUN7sfR0k7tAXmDIUTu6H3C8zMuJG/fLSRqxtF89aIWMKCbdxJYwrD7sdRvOx+HKVZtWbOCLsNr4dPHmbE0Vd4aUBz4n4+xt2T4zhxJt3XFRpjTL4sOHyhQhQMfc8ZJDF+Knes+zWT76zLpn2nGDxpBYeSLr6pvTGmfOrSpctF999Yv76YTy0uZtYv4itnB0ms0Ro+epAbTwzivT4TGLrwDAMnLuftMV2IqeydU+mMMaVHXFycr0soMtvj8LXWA2DMYhA/2n0xmIU37uf46XQGvbmchEOlZzguY0qjK+EYbUko6udowVEa1GwL476G2h1p8M3DLG3zJdmZmQx6czkb9p70dXXGlEohISEcPXrUwuMyqSpHjx4lJCSk0OvYWVWlSVYGfDYeVr1FSsx13HFoLPvSQnh7bBfa1InydXXGlCoZGRkkJiaSmmrHBC9XSEgIderUITAw8IL5eZ1VZcFRGq2ZCZ88QmZ4be5J/R0/ptW08DDGlDg7Hbcs6TAC7llEQFYKM/TP/CJoI8Mmx7Eu0e4maIzxPQuO0iqmM9y7BL+ouryU+Q+GBH5j4WGMKRUsOEqzyDow+jOkwXWMz5jA7wPeY9jkFRYexhifsuAo7UIinIsFO4xgZOY8/uU/gXsmL7PwMMb4jAVHWeAfCLe/Cj2e4uasb5ni9w8emPylhYcxxicsOMoKEbj2EbhzCm0lgdnyJE9M/sjCwxhT4iw4yprWA5ARC6gdnMJs/szzk2dZeBhjSpQFR1lUrxv+Y78kPLIyU3ia6ZNfsfAwxpQYC46yqkpjAsZ9BTXb8C9e4ovJf2LdnuO+rsoYcwWw4CjLwqoQPPoTUpvcxu95m81TxrFu9xFfV2WMKee8Ghwi0lNEtopIgog84WF5sIjMdZfHiUh9d360iCwVkWQReS1H+1ARWSgiW0Rko4j49g71pUFgBUKHzCSp4wPcxeecmDqADT/v9XVVxphyzGvBISL+wASgF9ACGCIiLXI1GwMcV9XGwEvAc+78VOBJ4FEPm/6XqjYD2gPdRaSXN+ovU/z8CL/9nxy/8Vm68yP+M25l09atvq7KGFNOeXOPozOQoKo7VDUdmAP0zdWmLzDDnZ4H9BARUdXTqroMJ0DOUdUzqrrUnU4H1gB1vPgeypRK1/+K431nUo/9VH63Fz+tW+Hrkowx5ZA3g6M2sCfH80R3nsc2qpoJnASiC7NxEYkCbge+uuxKy5Eq7W8naejH+KPU+rAfO1Z87OuSjDHljDeDQzzMyz2Ge2HaXLxhkQDgXeBVVd2RR5txIhIvIvGHDx8usNjypHrTzmSO/oIDUo06n41i38oPfV2SMaYc8WZwJAIxOZ7XAfbl1cYNg0jgWCG2PQnYpqov59VAVSepaqyqxlatWrVIhZcHNes2Jnjsp2yjPtUWjeV4/Dxfl2SMKSe8GRyrgCYi0kBEgoDBwIJcbRYAI93pAcASLeDOUiLyd5yAebiY6y13YmrXRkb+l/XamIhP7uVM/Lu+LskYUw54LTjcYxYPAouBzcB7qrpRRJ4WkT5usylAtIgkAI8A507ZFZGdwIvAKBFJFJEWIlIH+BPOWVprRGStiIz11nsoD1o0iCFt8PvEZ19FyCe/IiN+lq9LMsaUcXbr2CvEojXbCf/vSK71W0/WrS/h32m0r0syxpRyduvYK1zvDo3Y+cspLMlqh//C36ErJvq6JGNMGWXBcQUZfu1VrOv+GouzYpHPHof/verrkowxZZAFxxXmoVta8W27F/gkqyt88SR8+4KvSzLGlDEWHFcYEeGv/drxSeOnmZ91DSz5Oyz5B1wBx7qMMcXDguMKFODvx8tDY5lTazzvZ98A3z4PXz5l4WGMKRQLjitUSKA/k0Z1YWql3zFHfwn/ewU+G2/hYYwpUICvCzC+E1khkOljutJ/QhZZ6UHcHfcGZKVB73+Dn/1PYYzxzILjClc9IoSZY7sw4PVMNCCYYfFTITMd+rwKfv6+Ls8YUwpZcBgaVa3ItNFdGDIpG8KCGbb2bchKh35vgL/9iBhjLmR/FQwA7WKimDg8ljHTFf/oYIasn+6Ex52TwT/Q1+UZY0oRCw5zzvVNq/KvgW15eK4SWLsCAza9AdmZMHC6hYcx5hwLDnOBfu1rcyQ5jUcXQnDDYG7f8jJ8OM7Z87BjHsYYLDiMB2Ovbcjh5DR+8w1ENXuIaze+AgHB0Pd1O9vKGGPXcRjPnujZjP7tazN8Sxc2N/sN/PguLPydXedhjLE9DuOZiPDsnW1IPJFC3/VX803HLGqufh0CQqDnsyCe7vprjLkS2B6HyVNQgB9vDutIragK3LrhRk61GwdxE214EmOucBYcJl+VwoKYOqoTWQr9t99KWvt7nOFJvn7W16UZY3zEgsMUqGHVikwc1pGdR88w9tBdZLcbBt88C9+96OvSjDE+4NXgEJGeIrJVRBJE5AkPy4NFZK67PE5E6rvzo0VkqYgki8hrudbpKCLr3XVeFbHO9pLQrVE0/+zfmu+2H+PJ7HvR1gPhq7/C8td9XZoxpoR5LThExB+YAPQCWgBDRKRFrmZjgOOq2hh4CXjOnZ8KPAk86mHTbwDjgCbuo2fxV288GRQbw/3XN2L2yr1MrfIHaN4HFo+HVVN8XZoxpgR5c4+jM5CgqjtUNR2YA/TN1aYvMMOdngf0EBFR1dOqugwnQM4RkZpAhKouV1UFZgL9vPgeTC5/uOUqeraswd8/28aXLf4JTXvCwkfgh9m+Ls0YU0K8GRy1gT05nie68zy2UdVM4CQQXcA2EwvYpvEiPz/hpbva0bp2JL99fyMbu78KDW+EBQ/C+nm+Ls8YUwK8GRyejj3kPoezMG0uqb2IjBOReBGJP3z4cD6bNEVVIcifySNiiawQyJjZGzjQeyrUvdoZmmTTAl+XZ4zxMm8GRyIQk+N5HWBfXm1EJACIBI4VsM06BWwTAFWdpKqxqhpbtWrVIpZuClItIoQpIzuRlJrB2Hc3cmbgbKjdEeaNhp8+93V5xhgv8mZwrAKaiEgDEQkCBgO5/x1dAIx0pwcAS9xjFx6p6n4gSUS6umdTjQA+Kv7STWG0qBXBf4a2Z9O+Uzz8YQLZQ9+H6i1h7jDYvtTX5RljvMRrweEes3gQWAxsBt5T1Y0i8rSI9HGbTQGiRSQBeAQ4d8quiOwEXgRGiUhijjOyfgVMBhKA7cCn3noPpmC/aFadP9/ags83HeS5r/fD8PlQpQm8OwR2/s/X5RljvEDy+Qe/3IiNjdX4+Hhfl1FuqSp/+Wgjs1bs4tn+rRncogJMvxVO7YXh/4WYTr4u0RhzCURktarG5p5vV46byyYiPHV7C65rWpU//3cD/zsgMOIjqFgNZt8JBzf5ukRjTDGy4DDFIsDfj9eGtqdh1TB+9fZqElLDnb2NwFB4uz8c3+XrEo0xxcSCwxSbiJBApozsRFCAH6Onr+JYUE0Y9iFkpMCsOyDZTos2pjyw4DDFKqZyKJNGxHLgVCr3zYonLfoqGPoenNoHswdAWpKvSzTGXCYLDlPsOtStxL8HtmXVzuOM/2A9GtMZBs2EA+thzlDITPN1icaYy2DBYbzi9ra1+N1NTfnwh7289d0OaHoz9Hsdfv4WPrwXsrN8XaIx5hJZcBiv+W2PxtzauibPfLqFpVsOQdvBcMs/YdNHsPD3dhdBY8ooCw7jNSLCCwPb0LxGBL999wcSDiVDtwfgmt/B6mmw9J++LtEYcwksOIxXhQYF8NbIWIIC/Lh3Zjwnz2RAj6eg/XD49nmIe9PXJRpjisiCw3hd7agKTBzekcTjZ3jw3TVkZivc9jI0uw0+/YMNx25MGWPBYUpEp/qV+Xu/Vny37QjPfLoF/APgzilQ7xqYfx8kfOnrEo0xhWTBYUrMXZ3qMurq+kxZ9jPvxe+BwBAY8g5Uaw5zh0OijSdmTFlgwWFK1J9vbc41javw5/kbWL3rGIREOleXV6zuXCB4eKuvSzTGFMCCw5Sos2Na1YwK4b5Za9h3IsUZDHH4fPAPcoYmObGn4A0ZY3ymUMEhIg+JSIQ4pojIGhG52dvFmfIpKjSIySNiSc3IYtyseFLSs6ByAxj2gTMkydv94fRRX5dpjMlDYfc4RqvqKeBmoCpwD/Cs16oy5V6T6uG8MrgdG/ed4rF5P6KqUKM1DJnjjKT7zkBIS/Z1mcYYDwobHOJ+7Q1MU9Ufc8wz5pL0aF6dP9zSjE/W7ef1r7c7M+t3h4HTYN8P8N5wyEz3bZHGmIsUNjhWi8jnOMGxWETCgWzvlWWuFPdf35C+7WrxwuKtfL7xgDOz2a3Q5z+wfYlzqq6Na2VMqVLY4BiDcz/wTqp6BgjE6a4y5rKICM/d2YY2dSL53dy1bD3gDrvefhj88mnY+CEsetTGtTKmFClscHQDtqrqCREZBvwZOFnQSiLSU0S2ikiCiDzhYXmwiMx1l8eJSP0cy8a787eKyC055v9ORDaKyAYReVdEQgr5HkwpFRLoz6ThsYQGBzB25iqOn3a7p7o/5Dzip8KSv/u2SGPMOYUNjjeAMyLSFvgDsAuYmd8KIuIPTAB6AS2AISLSIlezMcBxVW0MvAQ8567bAhgMtAR6Aq+LiL+I1AZ+C8SqaivA321nyrgakSFMGt6Rg6fS+PXsNWRkuT2hN/0VOoyA7/4Fyyf4tkhjDFD44MhUVQX6Aq+o6itAeAHrdAYSVHWHqqYDc9z1c+oLzHCn5wE9RETc+XNUNU1VfwYS3O0BBAAVRCQACAX2FfI9mFKufd1KPHNHa5bvOMrfPtnkzBRxxrVq3gcW/xHWvuPbIo0xhQ6OJBEZDwwHFrp7E4EFrFMbyHklV6I7z2MbVc3E6f6KzmtdVd0L/AvYDewHTqrq555eXETGiUi8iMQfPmz3ui4r7uxYh3HXNWTm8l28E7fbmennD3dOhoY3wEcPwpaFvizRmCteYYPjLiAN53qOAzh/2F8oYB1Pp+vmPsKZVxuP80WkEs7eSAOgFhDmHnO5uLHqJFWNVdXYqlWrFlCqKU0e79mM65tW5S8fbSBuh3shYEAw3DUbarWH9++Bn7/zbZHGXMEKFRxuWMwGIkXkNiBVVfM9xoGzlxCT43kdLu5WOtfG7XqKBI7ls+5NwM+qelhVM4APgasL8x5M2eHvJ7w6pD11o0O5/+3V7D56xlkQXBHuft+5yvzdIc61HsaYElfYIUcGASuBgcAgIE5EBhSw2iqgiYg0EJEgnIPYC3K1WQCMdKcHAEvcYykLgMHuWVcNgCbu6+8GuopIqHsspAewuTDvwZQtkRUCmTKyE9kKo2es4lRqhrMgtLIzrlVoJXj7Tjj8k28LNeYKVNiuqj/hXMMxUlVH4ByofjK/FdxjFg8Ci3H+uL+nqhtF5GkR6eM2mwJEi0gC8AjOtSKo6kbgPWAT8BnwgKpmqWoczkH0NcB6t/5JhX63pkxpUCWMicM6svPIaR585wcyz55pFVELhv8XxA9m9bNBEY0pYaKFuLBKRNarauscz/2AH3POK81iY2M1Pt7u9VBWzV21m8c/WM/IbvX4a99W5xfsXwfTb3WGZB/9GYRV8V2RxpRDIrJaVWNzzy/sHsdnIrJYREaJyChgIbCoOAs0Ji93darLvdc2YMbyXcxcvvP8gpptYOhcOLnHGVE39ZSvSjTmilLYg+OP4XQJtQHaApNU9XFvFmZMTk/0as5Nzavx14838e1POU6vrnc1DJoJBzfCnKGQkeq7Io25QhT6Rk6q+oGqPqKqv1PV+d4sypjc/P2EVwa3p0m1ijwwew0Jh5LOL2x6C/SbCDuXwbx7ICvTd4UacwXINzhEJElETnl4JImI9QuYEhUWHMCUUZ0IDvRn9PR4jp3OMeR6m4HQ+wXYuggW/AaybfBmY7wl3+BQ1XBVjfDwCFfViJIq0pizakdV4K0RHTlwKpX7Z60mLTPHkOud74Ub/gg/vgOf/8lG1DXGS+ye46bMaV+3Ev8a2JaVO4/xxw83cMGZgdf/AbrcDyteh2//5bsijSnHAnxdgDGXok/bWuw4nMzLX26jcbWK/OqGRs4CEbjlGUg5Dkv/DhWinD0RY0yxseAwZdZDPZqw/fBpnl+8hYZVw7ilZQ1ngZ8f9J3gnJ676FHwC4BYu++YMcXFuqpMmSUivDCgDW3rRPHwnLVs2Jvj3mL+gTBoBjS5GT55GOKn+a5QY8oZCw5TpoUE+jNpREcqhQYydkY8B0/luI4jIBjuetvCw5hiZsFhyrxq4SFMGdWJU6kZ3DsznpT0HGdaWXgYU+wsOEy50LxmBK8Obs/6vSf5/ftryc7OcaaVhYcxxcqCw5QbN7Wozh97NWfR+gO89GWu4dYtPIwpNhYcplwZe20D7oqN4T9LEpj/Q+KFC3OHx+rpPqnRmLLOgsOUKyLC3/q1omvDyjw+bz2rdx27sEFAMAya5YTHxw9ZeBhzCSw4TLkTFODHxNDrhRYAABzPSURBVGEdqRUVwtgZ8SQcSr6wQWCIhYcxl8GCw5RLUaFBTL+nM/5+fgyfEsfeEykXNrDwMOaSWXCYcqt+lTBmju5Mclomw6fEcTQ57cIGFh7GXBKvBoeI9BSRrSKSICJPeFgeLCJz3eVxIlI/x7Lx7vytInJLjvlRIjJPRLaIyGYR6ebN92DKtha1IpgyshN7j6cwatoqktNy3avjbHg0/qUbHjN8U6gxZYjXgkNE/IEJQC+gBTBERFrkajYGOK6qjYGXgOfcdVsAg4GWQE/gdXd7AK8An6lqM5y7EW721nsw5UPnBpV5/e4ObNp/inEz40nNyLqwQWCIc7ZV41/Cx7+18DCmAN7c4+gMJKjqDlVNB+YAfXO16Quc/S2dB/QQEXHnz1HVNFX9GUgAOotIBHAdMAVAVdNV9YQX34MpJ3o0r84LA9rw/fajPDTnBzKzct3oycLDmELzZnDUBvbkeJ7ozvPYRlUzgZNAdD7rNgQOA9NE5AcRmSwiYd4p35Q3/TvU4S+3tWDxxoP8aX6u+3iAhYcxheTN4BAP83Lfki2vNnnNDwA6AG+oanvgNHDRsRMAERknIvEiEn/48OHCV23KtdHXNOA3v2jM3Pg9PPfZ1osbXBQe00u8RmNKO28GRyIQk+N5HWBfXm1EJACIBI7ls24ikKiqce78eThBchFVnaSqsaoaW7Vq1ct8K6Y8eeSXTbm7S10mfrOdN7/ZfnGDC8LjIedOgnYbWmPO8WZwrAKaiEgDEQnCOdi9IFebBcBId3oAsESd/oMFwGD3rKsGQBNgpaoeAPaIyFXuOj2ATV58D6YcEhGe7tuKW9vU5JlPt/Deqj0XNwoMgcHvQOtBsORvsOgxyM66uJ0xVyCv3QFQVTNF5EFgMeAPTFXVjSLyNBCvqgtwDnLPEpEEnD2Nwe66G0XkPZxQyAQeUNWzv7W/AWa7YbQDsFu7mSLz9xNeGtSOUykZPPHhOiJDA8/fQfCsgCC4400IrwHfvwrJB6H/W06oGHMFk4sOEJZDsbGxGh8f7+syTCl0Oi2TuyfHsWn/Kabf04mrG1Xx3HD5BFj8R6jX3dkTqRBVsoUa4wMislpVY3PPtyvHzRUtLDiAaaM6Ua9yKONmrmZ94knPDbs9AHdOgT0rYVovOLm3ZAs1phSx4DBXvEphQcwa04XICoGMnLaS7YeTPTdsPQCGzYMTe2DKzXBoS8kWakwpYcFhDFAjMoRZYzojwIgpK9l/MsVzw4Y3wD2LIDsDpt4Cu1eUYJXGlA4WHMa4GlatyIzRnTmZksHwKSs5fjrdc8OabWDM5xAaDTP7wpaFJVuoMT5mwWFMDq1qR/LWiFh2HzvDqOmrOJ17UMSzKtV3wqN6S5g7zG5Fa64oFhzG5NKtUTSvDWnP+sQTjJ0Rn3d4hFWBkR9D45ucW9F+/axdKGiuCBYcxnhwc8sa/HtQW+J+PsqwKXGcOJNHt1VQmHN6bru74etnnCvNs/IIGmPKCQsOY/JwR/s6vH53RzbuPcVdb67g0KlUzw39A6HvBLj2UVgzA94bDulnSrZYY0qQBYcx+ejZqgZTR3Viz/EzDHxzOXuO5REIItDjSej9L9j6KczqB2eOlWyxxpQQCw5jCnBNkyq8PbYLx0+nM2Di92w7mJR34873wqAZsG8tTO3pXPNhTDljwWFMIXSoW4n37u9GtsKgN5ezLjGf+4e16AvD50PSAXjrRti+pOQKNaYEWHAYU0jNakTw/n3dCAsOYOhbcazYcTTvxvW7n7/WY1Z/+OppO2huyg0LDmOKoH6VMObdfzU1IkMYOXUlX20+mHfjas3g3qXQYTh892+Yfqt1XZlywYLDmCKqERnCe/d146oa4dw3azUfrc1nwMOgUOjzH2eAxIMbYOI1sGVRyRVrjBdYcBhzCSqHBTF7bBc61qvEw3PXMmvFrvxXaD0A7vsWKtWDOUPg0ycgM61kijWmmFlwGHOJwkMCmTG6M7+4qhpP/ncDr3+dkP8K0Y1gzBfQ5X6IewOm/BKOerh1rTGlnAWHMZchJNCficM70rddLZ7/bCvPfLqZfG+OFhAMvZ5zrjY/vgvevB7Wzyu5go0pBhYcxlymQH8/XhrUjmFd6/LmNzv44/wNZGUXMGZVs1vh/mVQvQV8MAYW/MauNjdlhleDQ0R6ishWEUkQkSc8LA8Wkbnu8jgRqZ9j2Xh3/lYRuSXXev4i8oOIfOLN+o0pLD8/4W99W/HrGxrx7srdPDTnB9Izs/NfKSoGRi2Eax6BNbPgrV/Aoc0lU7Axl8FrwSEi/sAEoBfQAhgiIi1yNRsDHFfVxsBLwHPuui2AwUBLoCfwuru9sx4C7DfMlCoiwh96NmN8r2Z8sm4/42bFk5Kelf9K/oFw01Mw/EM4cwQm3QirZ9gou6ZU8+YeR2cgQVV3qGo6MAfom6tNX2CGOz0P6CEi4s6fo6ppqvozkOBuDxGpA9wKTPZi7cZcsvuub8Qz/VvzzU+HuXvyirzvJphTo1/A/f+DmM7w8W+d7qvUU94v1phL4M3gqA3kvNop0Z3nsY2qZgIngegC1n0Z+ANQQD+AMb4zpHNdXh/aga0Hkuj9yncs3Xqo4JXCqztDlfziz7BxPrx5Hexd4/1ijSkibwaHeJiXe/87rzYe54vIbcAhVV1d4IuLjBOReBGJP3z4cMHVGlPMerWuyce/uYbqESHcM20Vz322hcysAv7f8fOH6x6DUYsgKx0m94BPHrGRdk2p4s3gSARicjyvA+zLq42IBACRwLF81u0O9BGRnThdX78Qkbc9vbiqTlLVWFWNrVq16uW/G2MuQcOqFfnvA90Z0rkub3y9nSFvFbLrql43+NX/oNO9sHo6vNoeVr5l412ZUsGbwbEKaCIiDUQkCOdg94JcbRYAI93pAcASdU6CXwAMds+6agA0AVaq6nhVraOq9d3tLVHVYV58D8ZctpBAf57p35pXBrdj075T3PrqMr4uTNdVhUrQ+3nntN2abWDRo0731c/fer9oY/LhteBwj1k8CCzGOQPqPVXdKCJPi0gft9kUIFpEEoBHgCfcdTcC7wGbgM+AB1S1gNNTjCnd+rarzYLfXEO18GBGTVvF84XpugLnWo8RC+CutyE9CWbcDu+NgBO7vV+0MR5Ivle5lhOxsbEaHx/v6zKMASA1I4u/fryJd1fuplP9SvxnSAdqRIYUbuWMFPj+NVj2Img2dH8Iuj/sDKZoTDETkdWqGpt7vl05bkwJO9t19fJd7di47xS9X/2ucF1XAIEV4PrH4MFVztXn3zwHr3WCDR/atR+mxFhwGOMj/drX5uMcXVcvLC5k1xVAZB0YMBXu+RRCK8G8e2D6bXBgvXeLNgYLDmN8qtG5s65imLB0O0PfiuPAydTCb6De1TDuG7jtZTi0yTl4bqfvGi+z4DDGx5yuqza8fFc7Nuw7Se9Xv+Obn4pw7ZGfP8TeA79dA53HnT99N26Snb5rvMKCw5hSol/72ix48BqqVgxm5NSVReu6Auf03V7POdd/1GwLnz4Gb3SDNTMhowh7McYUwILDmFKkcTWn62pwJ6frauCby1mXeKJoG6nWHEZ8BHfNhoAQZ8j2l1vDty9YF5YpFnY6rjGl1Edr9/K3TzZxJDmdOzvU4Q89r6J6RCFP2z1L1blg8Pv/QMIXEBgK7YdB119B5YbeKdyUG3mdjmvBYUwplpSawYSl25m67GcC/IVfXd+Ie69rSEigf8Er53ZwEyyfAOvmgmZB89vh6t9CnYv+LhgDWHBYcJgybffRMzzz6WY+3XCA2lEVeLxXM25vUxPnLgRFlHQA4t6E+CmQehLqdoOrfwNNe4Gf9V6b8yw4LDhMObBix1Ge/ngTm/afomO9Sjx5WwvaxURd2sbSkuGHt2HFBGf4ksqN4OoHoe0Q50JDc8Wz4LDgMOVEVrbywepEnl+8lSPJafRvX5vHel5FzchL/GOflQmbFzjHQfatgdBo57TeTmMhrErxFm/KFAsOCw5TziSnZfL60gQmL/sZfxHuu74h913XiApBl3D8A5wD6bu+h+WvwdZFzhlZrQdAqwFQ/1rwDyjeN2BKPQsOCw5TTu05doZnP93CwvX7qRkZwuM9m9GnbS38/C7h+MdZh39yAmTDB5CeDKFVoEUfaHkH1OvuXHRoyj0LDgsOU86t/PkYT3+ykQ17T9EuJoq/3N6CDnUrXd5GM1Jg2xfOrWx/+gwyzkBYtfMhUrebhUg5ZsFhwWGuANnZygdrnOMfh5PS6N26BmOuaUCHupUu7QysnNLPwLbP3RBZDJkpULE6tOgLLftDTBc7K6ucseCw4DBXkOS0TCZ+vZ0Z3+8kKS2TVrUjGNmtPre3rXVp14Dkln7aCY+N850wyUyF8JrQop+zJ1Knk4VIOWDBYcFhrkCn0zL58Ie9zPx+J9sOJVM5LIjBnWK4u2s9akcV0ym3aUk5QuQLyEqDiNpOiDS/HWp3hICg4nktU6IsOCw4zBVMVfl++1Gmf7+TrzYfBODmFjUYeXV9ujasfPndWGelnnKOhWycDwlfQlY6BFSAmE7OmVn1ujtBEljEoVOMT1hwWHAYAzhnYb0dt4u5q/Zw4kwGzWqEM6Jbffq1r0VoUDGecpt60hkna+f/YOcyOLgBUPAPdrqy6nd3giSms11wWEr5JDhEpCfwCuAPTFbVZ3MtDwZmAh2Bo8BdqrrTXTYeGANkAb9V1cUiEuO2rwFkA5NU9ZWC6rDgMOZiKelZLPhxL9O/38Xm/aeICAngrk4xDO9an7rRXriHecpx2LUcdv0Pdn7n3K1Qs8E/yNkLqdfdCZOYLhAUVvyvb4qsxINDRPyBn4BfAonAKmCIqm7K0ebXQBtVvV9EBgN3qOpdItICeBfoDNQCvgSaAtWAmqq6RkTCgdVAv5zb9MSCw5i8qSrxu44z/fudfLbhANmq/OKqaoy8uj7XNqlSfN1YuaWehN0rnL2Rnctg/4/O4It+AVCrPdS/xjndt0YbCK8B3qrD5MkXwdEN+D9VvcV9Ph5AVZ/J0Wax22a5iAQAB4CqwBM52+Zsl+s1PgJeU9Uv8qvFgsOYwjlwMpXZcbt4d+VujiSnE1O5Aje3qMFNzasTW78Sgf5ePFMqLQl2x8GuZU731r41kO3ewTCsKtRo7YRIjdbOjaoqN7RrSLwsr+Dw5hgCtYE9OZ4nAl3yaqOqmSJyEoh256/ItW7tnCuKSH2gPRBXnEUbcyWrERnC72++igd/0ZiF6/az4Md9zFq+iynLfiYiJIAbm1XjpubVuf6qqkSEBBbviweHQ5ObnAc4p/zuW+t0aR1Y5zyWT4DsDGd5YBhUb+kGiRso1VrY8ZIS4M3g8LRfmXv3Jq82+a4rIhWBD4CHVfWUxxcXGQeMA6hbt25h6jXGuIID/OnfoQ79O9ThdFom3207zJebD7FkyyE+WruPAD+ha8NobmpejR7NqxNT2QvHRILCnGMe9bufn5eZDoe35AiT9bD+fWeIeADxhypNzwdJ9ZbOnklEHRtrqxiVua4qEQkEPgEWq+qLhanFuqqMKR5Z2coPu4/z5eZDfLn5IAmHkgFoViOcm5pX56YW1WlTO/LyxskqquxsOLHrfJDsd78m7Tvfxi8Qouo6IVK5IVRucH46qi4EBJdcvWWIL45xBOAcHO8B7MU5OD5UVTfmaPMA0DrHwfH+qjpIRFoC73D+4PhXQBOcM6lmAMdU9eHC1mLBYYx3/HzkNF9tPsiXmw+yaudxsrKVquHB9HC7tLo3rnLpo/VertNH4NBmOP4zHNvhPn52HulJORoKRMa4YZIjUCo1gEr1Ibiib+ovBXx1Om5v4GWc03Gnquo/RORpIF5VF4hICDAL51jFMWCwqu5w1/0TMBrIxOmS+lRErgG+A9bjhAjAH1V1UX51WHAY430nzqTz9dbDfLn5IN9sPUxSWiYBfkKLWhG0i4mifd0o2sVUon50qPfO1CoMVSdUzgVKjmA5/jOcOXph++AIZziV8BoQUcuZjqjlPA+vBRE1nYEfy2FXmF0AaMFhTIlJz8xm5c/H+H77EdbuOcGPe05wOj0LgEqhgbSNiaJ9TCXa1Y2iXZ0oIkOL+UD75Ug54YbKz04XWNIBOLUPkvbDqf2QfOD82V5niZ8THhE1z4dJeA0nZEKrODfHCot2vgZHlJlTiy04LDiM8ZmsbCXhUDI/7D7O2j0n+GH3CX46lMTZPz8Nq4adC5L2MVE0qxFOgDdP/b0c2dlw5sj5MDkbKEn73K8HnOmU457X9wt0g6QKhFbOESzu17OPsCpQoTKERDpnivkgbCw4LDiMKVWSUjNYn3iSH9wgWbvnOEeS0wEICfSjTe0oWteJpHG1is6jakUqhZWhwRIzUiD5oNP1deaY0z125ojz/PQRZ17O56kn8t6WX6ATIBc8InI9j7rwebC7PLzmJY9UbMFhwWFMqaaqJB5P4Yc9J1jrBsmm/adIzcg+1yY6LIhGbpA0ORso1SpSIyLEt8dNikNWJqQcyxEsR51H2innKvs8H6ece6Pk5Y/7IejSTpf2xQWAxhhTaCJCTOVQYiqH0qdtLcC5MdXeEykkHEo+/ziczMJ1+zmZknFu3YrBATSqGnYuVBpXdb7WrRxaeru8cvMPgIrVnEdRZaY5AXIuTE6cn/bCBZG2x2GMKXNUlSPJ6eeCZHuOYDlwKvVcO38/oUZECLWiQqgVVYHaURUu+ForKoTw4r4CvhyxPQ5jTLkhIlQND6ZqeDDdGkVfsCwpNYPth0+TcCiZnUdOs+9ECntPpLBm93EWrttPZvaF/yyHhwRcECQ5g6VGRAhVw4OL566J5YgFhzGmXAkPCaRdTBTtYqIuWpaVrRxJTmPviRT2nXukknjcmV6z+zgnzmRctF5YkD9VwoOJDgsiumIwVSoGUaXi+efRFYOoWjGY6IrBRFUILNkr533AgsMYc8Xw9xOqR4RQPSKEDnUreWxzOi2T/SdT2HsilQMnUziSnM7R5HSOnk7jSHIae46d4YfdJzh2Oo1sDz39/n5CpdAgqlQMonJYEJEVAokKDSSiQiCR+TzCQwLxLyOBY8FhjDE5hAUH0LhaOI2rhefbLjtbOX4mnaOn0zmSnMbR5PNfnZBJ59jpdLYdSuZkSgYnUzJIz8zOc3sizkH+3IESFhxAxbOPkBzTwQGEBQcQHhJwQZuQQD+vn2FmwWGMMZfAz0/cbqpgmlbPP2TOSs3IOhciJ85knJs++ziV63nCoWROp2WSlJbJ6bRMj3s4ufn7CWFB/oSHBFIxOID/PtC92McLs+AwxpgSEhLoT0igP9UjQoq8rqqSkpFFclomyamZnE7LIiktg9NpWSSnZZCcluXOzyQ5LZMkdzo4oPhPR7bgMMaYMkBECA0KIDQogAJ60byujFwZY4wxprSw4DDGGFMkFhzGGGOKxILDGGNMkVhwGGOMKRILDmOMMUViwWGMMaZILDiMMcYUyRVxPw4ROQzsusTVqwBHirGc4mJ1FY3VVTRWV9GU17rqqWrV3DOviOC4HCIS7+lGJr5mdRWN1VU0VlfRXGl1WVeVMcaYIrHgMMYYUyQWHAWb5OsC8mB1FY3VVTRWV9FcUXXZMQ5jjDFFYnscxhhjisSCwyUiPUVkq4gkiMgTHpYHi8hcd3mciNQvgZpiRGSpiGwWkY0i8pCHNjeIyEkRWes+/uLtutzX3Ski693XjPewXETkVffzWiciHUqgpqtyfA5rReSUiDycq02JfF4iMlVEDonIhhzzKovIFyKyzf3q8abXIjLSbbNNREaWQF0viMgW9/s0X0Si8lg33++5F+r6PxHZm+N71TuPdfP93fVCXXNz1LRTRNbmsa43Py+PfxtK7GdMVa/4B+APbAcaAkHAj0CLXG1+DUx0pwcDc0ugrppAB3c6HPjJQ103AJ/44DPbCVTJZ3lv4FNAgK5AnA++pwdwzkMv8c8LuA7oAGzIMe954Al3+gngOQ/rVQZ2uF8rudOVvFzXzUCAO/2cp7oK8z33Ql3/BzxaiO9zvr+7xV1XruX/Bv7ig8/L49+GkvoZsz0OR2cgQVV3qGo6MAfom6tNX2CGOz0P6CFeviO8qu5X1TXudBKwGajtzdcsRn2BmepYAUSJSM0SfP0ewHZVvdQLPy+Lqn4LHMs1O+fP0Aygn4dVbwG+UNVjqnoc+ALo6c26VPVzVc10n64A6hTX611OXYVUmN9dr9Tl/v4PAt4trtcrrHz+NpTIz5gFh6M2sCfH80Qu/gN9ro37S3YSiC6R6gC3a6w9EOdhcTcR+VFEPhWRliVUkgKfi8hqERnnYXlhPlNvGkzev9C++LwAqqvqfnB+8YFqHtr4+nMbjbOn6ElB33NveNDtQpuaR7eLLz+va4GDqrotj+Ul8nnl+ttQIj9jFhwOT3sOuU83K0wbrxCRisAHwMOqeirX4jU43TFtgf8A/y2JmoDuqtoB6AU8ICLX5Vruy88rCOgDvO9hsa8+r8Ly5ef2JyATmJ1Hk4K+58XtDaAR0A7Yj9MtlJvPPi9gCPnvbXj98yrgb0Oeq3mYV6TPzILDkQjE5HheB9iXVxsRCQAiubRd6yIRkUCcH4zZqvph7uWqekpVk93pRUCgiFTxdl2qus/9egiYj9NlkFNhPlNv6QWsUdWDuRf46vNyHTzbXed+PeShjU8+N/cA6W3A3ep2hOdWiO95sVLVg6qaparZwFt5vJ6vPq8AoD8wN6823v688vjbUCI/YxYcjlVAExFp4P63OhhYkKvNAuDs2QcDgCV5/YIVF7cPdQqwWVVfzKNNjbPHWkSkM8739KiX6woTkfCz0zgHVzfkarYAGCGOrsDJs7vQJSDP/wR98XnlkPNnaCTwkYc2i4GbRaSS2zVzszvPa0SkJ/A40EdVz+TRpjDf8+KuK+cxsTvyeL3C/O56w03AFlVN9LTQ259XPn8bSuZnzBtH/MviA+csoJ9wztD4kzvvaZxfJoAQnK6PBGAl0LAEaroGZxdyHbDWffQG7gfud9s8CGzEOZtkBXB1CdTV0H29H93XPvt55axLgAnu57keiC2h72MoThBE5phX4p8XTnDtBzJw/sMbg3NM7Ctgm/u1sts2FpicY93R7s9ZAnBPCdSVgNPnffZn7OzZg7WARfl9z71c1yz3Z2cdzh/Emrnrcp9f9Lvrzbrc+dPP/kzlaFuSn1defxtK5GfMrhw3xhhTJNZVZYwxpkgsOIwxxhSJBYcxxpgiseAwxhhTJBYcxhhjisSCw5hSTJzRfD/xdR3G5GTBYYwxpkgsOIwpBiIyTERWuvdeeFNE/EUkWUT+LSJrROQrEanqtm0nIivk/P0vKrnzG4vIl+4AjGtEpJG7+YoiMk+ce2bM9vaozMYUxILDmMskIs2Bu3AGtWsHZAF3A2E4Y2Z1AL4BnnJXmQk8rqptcK6MPjt/NjBBnQEYr8a5YhmckU8fxrnfQkOgu9fflDH5CPB1AcaUAz2AjsAqd2egAs7gctmcHwTvbeBDEYkEolT1G3f+DOB9d1yj2qo6H0BVUwHc7a1Ud0wkce42Vx9Y5v23ZYxnFhzGXD4BZqjq+AtmijyZq11+4/vk1/2UlmM6C/u9NT5mXVXGXL6vgAEiUg3O3fe5Hs7v1wC3zVBgmaqeBI6LyLXu/OHAN+rcSyFRRPq52wgWkdASfRfGFJL952LMZVLVTSLyZ5y7vfnhjKT6AHAaaCkiq3HuGHmXu8pIYKIbDDuAe9z5w4E3ReRpdxsDS/BtGFNoNjquMV4iIsmqWtHXdRhT3KyryhhjTJHYHocxxpgisT0OY4wxRWLBYYwxpkgsOIwxxhSJBYcxxpgiseAwxhhTJBYcxhhjiuT/AZ2/3Ql0I0+JAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torch.nn import init\n",
    "import numpy as np\n",
    "from IPython import display\n",
    "import torch.utils.data as Data\n",
    "\n",
    "num_inputs = 500\n",
    "num_examples = 10000\n",
    "true_w = torch.ones(500,1)*0.0056\n",
    "true_b = 0.028\n",
    "#随机生成的数据样本\n",
    "features = torch.tensor(np.random.normal(0, 1, (num_examples, num_inputs)), dtype=torch.float)#行*列=10000*500\n",
    "labels = torch.mm(features,true_w) + true_b\n",
    "labels += torch.tensor(np.random.normal(0, 0.01, size=labels.size()), dtype=torch.float) #扰动项\n",
    "#训练集和测试集上的样本&标签数----真实的特征和样本\n",
    "trainfeatures = features[:7000]\n",
    "trainlabels = labels[:7000]\n",
    "testfeatures = features[7000:]  \n",
    "testlabels = labels[7000:]\n",
    "print(trainfeatures.shape,trainlabels.shape,testfeatures.shape,testlabels.shape)\n",
    "\n",
    "#获得数据迭代器\n",
    "batch_size = 50 # 设置小批量大小\n",
    "def load_array(data_arrays, batch_size, is_train=True):  #自定义函数\n",
    "    #\"\"\"构造一个PyTorch数据迭代器。\"\"\"\n",
    "    dataset = Data.TensorDataset(*data_arrays)#features 和 labels作为list传入，得到PyTorch的一个数据集\n",
    "    return Data.DataLoader(dataset, batch_size, shuffle=is_train,num_workers=0)#返回的是实例化后的DataLoader\n",
    "train_iter = load_array([trainfeatures,trainlabels],batch_size)\n",
    "test_iter = load_array([testfeatures,testlabels],batch_size)\n",
    "\n",
    "#实现FlattenLayer层：将数据展平\n",
    "class FlattenLayer (nn.Module):\n",
    "    def _init_ (self):\n",
    "        super(FlattenLayer,self)._init_()\n",
    "    def forward (self,x):\n",
    "        return x.view(x.shape[0],-1)\n",
    "    \n",
    "#模型定义和参数初始化\n",
    "# num_inputs = 500\n",
    "num_hiddens = 256\n",
    "num_outputs = 1\n",
    "net = nn.Sequential(\n",
    "        FlattenLayer(),#输入层\n",
    "        nn.Linear(num_inputs,num_hiddens),#隐藏层\n",
    "        nn.ReLU(),#隐藏层激活函数Relu\n",
    "        nn.Linear(num_hiddens,num_outputs),#输出层\n",
    "        )\n",
    "for param in net.parameters():\n",
    "    print (param.shape)\n",
    "    init.normal_(param,mean=0,std=0.01)\n",
    "net.parameters()\n",
    "lr=0.01\n",
    "loss = torch.nn.MSELoss()\n",
    "optimizer = torch.optim.SGD(net.parameters(),lr)\n",
    "#记录列表（list），存储训练集和测试集上经过每一轮次，loss的变化\n",
    "def train1 (net,train_iter,test_iter,loss,num_epochs,batch_size,params = None,lr=None,optimizer=None):\n",
    "    train_loss=[]\n",
    "    test_loss=[]\n",
    "    for epoch in range(num_epochs):#外循环控制循环轮次\n",
    "        #step1在训练集上，进行小批量梯度下降更新参数\n",
    "        for X,y in train_iter:#内循环控制训练批次\n",
    "            y_hat = net(X)\n",
    "            l = loss(y_hat,y)#l.size = torch.Size([]),即说明loss为表示*标量*的tensor`\n",
    "            #梯度清零\n",
    "            if optimizer is not None:\n",
    "                optimizer.zero_grad()\n",
    "            elif params is not None and params[0].grad is not None:\n",
    "                for param in params:\n",
    "                    param.grad.data.zero_()\n",
    "            l.backward()\n",
    "            if optimizer is None:\n",
    "                SGD(params,lr,batch_size)\n",
    "            else:\n",
    "                optimizer.step()\n",
    "        #step2 每经过一个轮次的训练， 记录训练集和测试集上的loss\n",
    "        train_labels = trainlabels.view(-1,1)  \n",
    "        test_labels = testlabels.view(-1,1) \n",
    "        train_loss.append((loss(net(trainfeatures),train_labels)).item())#loss本身就默认了取平均值！\n",
    "        test_loss.append((loss(net(testfeatures),test_labels)).item())\n",
    "        print(\"epoch %d,train_loss %.6f,test_loss %.6f\"%(epoch+1,train_loss[epoch],test_loss[epoch])) \n",
    "    return train_loss, test_loss\n",
    "lr=0.01\n",
    "num_epochs = 20\n",
    "#batch_size、params epc已经定义\n",
    "train_loss, test_loss = train1 (net,train_iter,test_iter,loss,num_epochs,batch_size,net.parameters(),lr,optimizer)#每一给optimizer,默认None\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "x=np.linspace(0,len(train_loss),len(train_loss))\n",
    "plt.plot(x,train_loss,label=\"train_loss\",linewidth=1.5)\n",
    "plt.plot(x,test_loss,label=\"test_loss\",linewidth=1.5)\n",
    "plt.xlabel(\"epoch\")\n",
    "plt.ylabel(\"loss\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([14000, 200]) torch.Size([14000, 1]) torch.Size([6000, 200]) torch.Size([6000, 1])\n",
      "epoch 1,train_loss 0.015323,test_loss 0.014876\n",
      "epoch 2,train_loss 0.014219,test_loss 0.013920\n",
      "epoch 3,train_loss 0.013122,test_loss 0.012973\n",
      "epoch 4,train_loss 0.011951,test_loss 0.011959\n",
      "epoch 5,train_loss 0.010661,test_loss 0.010823\n",
      "epoch 6,train_loss 0.009283,test_loss 0.009612\n",
      "epoch 7,train_loss 0.007856,test_loss 0.008333\n",
      "epoch 8,train_loss 0.006466,test_loss 0.007070\n",
      "epoch 9,train_loss 0.005196,test_loss 0.005899\n",
      "epoch 10,train_loss 0.004102,test_loss 0.004848\n",
      "epoch 11,train_loss 0.003223,test_loss 0.003983\n",
      "epoch 12,train_loss 0.002556,test_loss 0.003301\n",
      "epoch 13,train_loss 0.002059,test_loss 0.002771\n",
      "epoch 14,train_loss 0.001703,test_loss 0.002377\n",
      "epoch 15,train_loss 0.001449,test_loss 0.002088\n",
      "epoch 16,train_loss 0.001262,test_loss 0.001867\n",
      "epoch 17,train_loss 0.001124,test_loss 0.001703\n",
      "epoch 18,train_loss 0.001023,test_loss 0.001581\n",
      "epoch 19,train_loss 0.000945,test_loss 0.001488\n",
      "epoch 20,train_loss 0.000883,test_loss 0.001415\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dfXQc9X3v8fd3Je3KWhnbWpkUbIidAkkggA3GhBKT9lKITWjMM05KwCkcX9rQm5x74MacNjT4pvdAHyDJjQshwRSc3EBiYuI2Tk0CJISbYLDBNJiHWrjOtTAPtuQnSZYl7X7vHzOS1utZefUwktj9vM7Zs7Mzv5n5avbhq/nN/H4/c3dEREQKJcY6ABERGZ+UIEREJJIShIiIRFKCEBGRSEoQIiISqXqsAxgpjY2NPmPGjLEOQ0TkPWXjxo273H1q1LKySRAzZsxgw4YNYx2GiMh7ipn9rtgyVTGJiEgkJQgREYmkBCEiIpHK5hqEiJSf7u5umpub6ezsHOtQ3vNqa2uZPn06NTU1Ja+jBCEi41ZzczMTJ05kxowZmNlYh/Oe5e60tLTQ3NzMzJkzS15PVUwiMm51dnaSyWSUHIbJzMhkMoM+E1OCEJFxTclhZAzlOFZ8gtix5wB3Pf4623a1j3UoIiLjSsUniNb2Lr7xZBOvvb1/rEMRERlXKj5BNNangCBRiIgU2rNnD//0T/806PUuuugi9uzZM+j1Fi9ezKpVqwa9XhwqPkFMSQe3fLW0HRzjSERkPCqWILLZ7IDrrV27lsmTJ8cV1qio+NtcU9VVTKytpkVnECLj2u3/splXduwb0W2efOxR/M2fnDJgmaVLl/LGG28wa9YsampqqK+v55hjjmHTpk288sorXHLJJWzfvp3Ozk6+8IUvsGTJEqC/f7i2tjYWLFjAxz72MX79618zbdo0fvzjHzNhwoQjxvfEE09w880309PTw1lnncU999xDKpVi6dKlrFmzhurqai688EL+4R/+gR/+8IfcfvvtVFVVMWnSJJ5++ulhH5+KTxAAmXRSCUJEIt1xxx28/PLLbNq0iV/84hd88pOf5OWXX+5rT7BixQoaGho4cOAAZ511FpdffjmZTOaQbWzZsoXvf//7fPvb3+aqq67i0Ucf5Zprrhlwv52dnSxevJgnnniCk046iWuvvZZ77rmHa6+9ltWrV/Paa69hZn3VWMuWLWPdunVMmzZtSFVbUWJNEGY2H/g6UAV8x93vKFieAh4CzgRagKvdfZuZzQBeBV4Piz7r7jfGFWemPkVru6qYRMazI/2nP1rmzp17SGOzb3zjG6xevRqA7du3s2XLlsMSxMyZM5k1axYAZ555Jtu2bTvifl5//XVmzpzJSSedBMB1113H8uXLuemmm6itreWGG27gk5/8JBdffDEA5557LosXL+aqq67isssuG4k/Nb5rEGZWBSwHFgAnA582s5MLil0P7Hb3E4C7gTvzlr3h7rPCR2zJAaAhnaSlTWcQInJk6XS6b/oXv/gFP//5z/nNb37DSy+9xOzZsyMbo6VSqb7pqqoqenp6jrgfd4+cX11dzXPPPcfll1/OY489xvz58wG49957+epXv8r27duZNWsWLS0tg/3TDhPnReq5QJO7b3X3LuBhYGFBmYXAg+H0KuB8G4NWMY31qmISkWgTJ05k//7o2+D37t3LlClTqKur47XXXuPZZ58dsf1+6EMfYtu2bTQ1NQGwcuVKPv7xj9PW1sbevXu56KKL+NrXvsamTZsAeOONNzj77LNZtmwZjY2NbN++fdgxxFnFNA3Ij7AZOLtYGXfvMbO9QO+52UwzexHYB/y1u/+qcAdmtgRYAnD88ccPOdCGdJLd7V3kck4ioVabItIvk8lw7rnn8pGPfIQJEybwvve9r2/Z/PnzuffeeznttNP44Ac/yEc/+tER229tbS0PPPAAV155Zd9F6htvvJHW1lYWLlxIZ2cn7s7dd98NwC233MKWLVtwd84//3xOP/30YcdgxU5jhr1hsyuBT7j7DeHrzwJz3f0v88psDss0h6/fIDjzaAPq3b3FzM4EHgNOcfeitzDMmTPHhzqi3Ipn/pNl//oKm267gMl1ySFtQ0RG3quvvsqHP/zhsQ6jbEQdTzPb6O5zosrHWcXUDByX93o6sKNYGTOrBiYBre5+0N1bANx9I/AGcFJcgWbqg6SgaiYRkX5xJojngRPNbKaZJYFFwJqCMmuA68LpK4An3d3NbGp4kRsz+wBwIrA1rkAz6eACki5Ui8ho+fznP8+sWbMOeTzwwANjHdYhYrsGEV5TuAlYR3Cb6wp332xmy4AN7r4GuB9YaWZNQCtBEgE4D1hmZj1AFrjR3VvjirUhHZxB6FZXERkty5cvH+sQjijWdhDuvhZYWzDvtrzpTuDKiPUeBR6NM7Z8jWEV0y6dQYiI9Kn4vpgApvSdQShBiIj0UoIAaqoSTJpQow77RETyKEGE1B+TiMihlCBC6m5DRKIMdTwIgK997Wt0dHQMWGbGjBns2rVrSNuPmxJEKFOf1DUIETlM3AliPFN336GGdIqNv9s91mGISDE/XQpv/3Zkt/l7p8KCOwYskj8exAUXXMDRRx/ND37wAw4ePMill17K7bffTnt7O1dddRXNzc1ks1m+/OUv884777Bjxw7+6I/+iMbGRp566qkjhnPXXXexYsUKAG644Qa++MUvRm776quvjhwTYqQpQYQawzMI9cckIvnyx4N4/PHHWbVqFc899xzuzqc+9Smefvppdu7cybHHHstPfvITIOjEb9KkSdx111089dRTNDY2HnE/Gzdu5IEHHmD9+vW4O2effTYf//jH2bp162Hbbm1tjRwTYqQpQYQa0klyDnsOdPc1nBORceQI/+mPhscff5zHH3+c2bNnA9DW1saWLVuYN28eN998M1/60pe4+OKLmTdv3qC3/cwzz3DppZf2dSd+2WWX8atf/Yr58+cftu2enp7IMSFGmq5BhDL1QXcbak0tIsW4O7feeiubNm1i06ZNNDU1cf3113PSSSexceNGTj31VG699VaWLVs2pG1Hidp2sTEhRpoSRCiTVmtqETlc/ngQn/jEJ1ixYgVtbW0AvPnmm7z77rvs2LGDuro6rrnmGm6++WZeeOGFw9Y9kvPOO4/HHnuMjo4O2tvbWb16NfPmzYvcdrExIUaaqphCvT266k4mEcmXPx7EggUL+MxnPsM555wDQH19Pd/97ndpamrilltuIZFIUFNTwz333APAkiVLWLBgAcccc8wRL1KfccYZLF68mLlz5wLBRerZs2ezbt26w7a9f//+yDEhRlps40GMtuGMBwHw7v5O5v7tE/zPSz7CZz/6/hGMTESGSuNBjKzxNB7Ee0pDOFCQutsQEQmoiilUXZVgcl2NqphEJBZnn302Bw8e+g/oypUrOfXUU8cooiNTgsiTUXcbIuOOu2P23m+btH79+jHd/1AuJ6iKKU8mnaJFt7mKjBu1tbW0tLQM6cdN+rk7LS0t1NbWDmo9nUHkydQnaXq3bazDEJHQ9OnTaW5uZufOnWMdyntebW0t06dPH9Q6ShB5GtLqsE9kPKmpqWHmzJljHUbFUhVTnkx9itaOLrI5nc6KiChB5Mmkk7jDng6dRYiIKEHk6W1NrZHlRESUIA7R24urbnUVEVGCOEQmHfToqltdRUSUIA6hDvtERPopQeSZUpfETF1+i4iAEsQhqhLGlLqkBg0SEUEJ4jAN6o9JRARQgjhMJp3Uba4iIihBHCZTn9SYECIixJwgzGy+mb1uZk1mtjRiecrMHgmXrzezGQXLjzezNjO7Oc4482XSKd3FJCJCjAnCzKqA5cAC4GTg02Z2ckGx64Hd7n4CcDdwZ8Hyu4GfxhVjlIZ0kj0HuunJ5kZztyIi406cZxBzgSZ33+ruXcDDwMKCMguBB8PpVcD5Fo4MYmaXAFuBzTHGeJjG+qA/pt0d3aO5WxGRcSfOBDEN2J73ujmcF1nG3XuAvUDGzNLAl4DbB9qBmS0xsw1mtmGk+otvCFtTq5pJRCpdnAkiaozAwn60i5W5Hbjb3Qccvcfd73P3Oe4+Z+rUqUMM81B9HfbpQrWIVLg4BwxqBo7Lez0d2FGkTLOZVQOTgFbgbOAKM/s7YDKQM7NOd/9mjPECwW2uoB5dRUTiTBDPAyea2UzgTWAR8JmCMmuA64DfAFcAT3ow+Oy83gJm9hWgbTSSAwSDBoHOIEREYksQ7t5jZjcB64AqYIW7bzazZcAGd18D3A+sNLMmgjOHRXHFU6rJE2pImK5BiIjEOia1u68F1hbMuy1vuhO48gjb+EoswRWRSBgN6SS7lCBEpMKpJXWEhnSSVvXHJCIVTgkiQiad0qBBIlLxlCAiNNSrwz4RESWICBl1+S0iogQRJZNOsfdAN93qj0lEKpgSRISGsDX1blUziUgFU4KI0KjW1CIiShBRGnoThK5DiEgFU4KI0Nfdhm51FZEKpgQRobfDPnW3ISKVTAkiwqQJNVQlTFVMIlLRlCAiJBLGlDo1lhORyqYEUURjfVJdfotIRVOCKKIhndQ1CBGpaEoQRWTqU6piEpGKpgRRRNAfk6qYRKRyKUEUkUkn2dfZQ1eP+mMSkcqkBFFEX39MHapmEpHKpARRRCYdtKbepWomEalQShBFZOrVmlpEKpsSRBEZddgnIhVOCaKI3iom3eoqIpVKCaKIoyZUU50w3eoqIhVLCaIIM1NrahGpaEoQA2hIJ9mlaxAiUqGUIAbQWJ+iVYMGiUiFUoIYQENaXX6LSOVSghhApj5Jq6qYRKRCKUEMIJNOsv9gDwd7smMdiojIqIs1QZjZfDN73cyazGxpxPKUmT0SLl9vZjPC+XPNbFP4eMnMLo0zzmIy9UFbCN3JJCKVKLYEYWZVwHJgAXAy8GkzO7mg2PXAbnc/AbgbuDOc/zIwx91nAfOBb5lZdVyxFtOg1tQiUsHiPIOYCzS5+1Z37wIeBhYWlFkIPBhOrwLONzNz9w537wnn1wIeY5xFNYb9MelCtYhUojgTxDRge97r5nBeZJkwIewFMgBmdraZbQZ+C9yYlzD6mNkSM9tgZht27tw54n9AQ7q3ikm3uopI5YkzQVjEvMIzgaJl3H29u58CnAXcama1hxV0v8/d57j7nKlTpw474EK9PbqqiklEKlGcCaIZOC7v9XRgR7Ey4TWGSUBrfgF3fxVoBz4SW6RFTExVU1NlqmISkYoUZ4J4HjjRzGaaWRJYBKwpKLMGuC6cvgJ40t09XKcawMzeD3wQ2BZjrJHMjEw6pQ77RKQixXZnkLv3mNlNwDqgCljh7pvNbBmwwd3XAPcDK82sieDMYVG4+seApWbWDeSAv3D3XXHFOhB12CcilSrWW0fdfS2wtmDebXnTncCVEeutBFbGGVupMvXqsE9EKpNaUh9BRmcQIlKhlCCOIFOvaxAiUpmUII6gIZ2kvStLZ7f6YxKRylJSgjCzL5jZURa438xeMLML4w5uPMik1ZpaRCpTqWcQf+bu+4ALganA54A7YotqHOnrsE8XqkWkwpSaIHpbPF8EPODuLxHdCrrs9HbYt0vdbYhIhSk1QWw0s8cJEsQ6M5tI0D6h7PV22KczCBGpNKW2g7gemAVsdfcOM2sgqGYqe71nELrVVUQqTalnEOcAr7v7HjO7Bvhrgp5Xy159qppkdUJVTCJScUpNEPcAHWZ2OvA/gN8BD8UW1TgS9MeksalFpPKUmiB63N0JBvj5urt/HZgYX1jjS6Y+qdtcRaTilHoNYr+Z3Qp8FpgXDidaE19Y40tDOqUEISIVp9QziKuBgwTtId4mGAnu72OLapxpTCfV3YaIVJySEkSYFL4HTDKzi4FOd6+IaxCgLr9FpDKV2tXGVcBzBF1zXwWsN7Mr4gxsPMnUp+joynKgS/0xiUjlKPUaxF8BZ7n7uwBmNhX4ObAqrsDGk/7+mA4yPVk3xtGIiIyOUq9BJHqTQ6hlEOu+52XC1tQtutVVRCpIqWcQ/2Zm64Dvh6+vpmCkuHKm1tQiUolKShDufouZXQ6cS9BJ333uvjrWyMaRxrBH1126k0lEKkjJY1K7+6PAozHGMm7pDEJEKtGACcLM9gMetQhwdz8qlqjGmbpkFbU1CTWWE5GKMmCCcPeK6U5jIEF/TCldpBaRilIxdyINV0M6SYt6dBWRCqIEUaJMvVpTi0hlUYIoUUM6qSomEakoShAlaqxPqYpJRCqKEkSJGtJJOrtzdHT1jHUoIiKjQgmiRH39MamaSUQqhBJEifr6Y9KFahGpELEmCDObb2avm1mTmS2NWJ4ys0fC5evNbEY4/wIz22hmvw2f/0uccZYikw6629DAQSJSKWJLEOGwpMuBBcDJwKfN7OSCYtcDu939BOBu4M5w/i7gT9z9VOA6YGVccZaqIa0zCBGpLHGeQcwFmtx9q7t3AQ8DCwvKLAQeDKdXAeebmbn7i+6+I5y/Gag1s1SMsR6RuvwWkUoTZ4KYBmzPe90czoss4+49wF4gU1DmcuBFdz+sbsfMlpjZBjPbsHPnzhELPEpdspoJNVW06lZXEakQcSYIi5hX2PHfgGXM7BSCaqf/GrUDd7/P3ee4+5ypU6cOOdBSZerVWE5EKkecCaIZOC7v9XRgR7EyZlYNTAJaw9fTgdXAte7+RoxxliyTTuoahIhUjDgTxPPAiWY208ySwCJgTUGZNQQXoQGuAJ50dzezycBPgFvd/f/GGOOgZNSaWkQqSGwJIrymcBOwDngV+IG7bzazZWb2qbDY/UDGzJqA/w703gp7E3AC8GUz2xQ+jo4r1lI1pJO0qopJRCpEySPKDYW7r6Vg7Gp3vy1vuhO4MmK9rwJfjTO2ocjUJ9nV3oW7YxZ1+UREpHyoJfUgZNJJunpytHdlxzoUEZHYKUEMglpTi0glUYIYhAb1xyQiFUQJYhDUo6uIVBIliEHI1AdVTGpNLSKVQAliEDLqsE9EKogSxCDU1lSRTlapiklEKoISxCA11Cdp1RmEiFQAJYhByqRT7NJtriJSAZQgBimT1hmEiFQGJYhBUpffIlIplCAGqSGdojXsj0lEpJwpQQxSY32SrmyO/Qd7xjoUEZFYKUEMUkPYFkLdfotIuVOCGKTe1tQaOEhEyp0SxCCpPyYRqRRKEIOUUY+uIlIhlCAGqe8ahBKEiJQ5JYhBSlVXMTFVrdbUIlL2lCCGQP0xiUglUIIYgoa0WlOLSPlTghiCTDqli9QiUvaUIIYgk07SomsQIlLmlCCGIFOfZHeH+mMSkfKmBDEEDekk3VlnX6f6YxKR8qUEMQSNvd1tqJpJRMqYEsQQqLGciFQCJYgh6O1uY5dudRWRMqYEMQSZdFDFpDMIESlnsSYIM5tvZq+bWZOZLY1YnjKzR8Ll681sRjg/Y2ZPmVmbmX0zzhiHoqGvR1ddgxCR8hVbgjCzKmA5sAA4Gfi0mZ1cUOx6YLe7nwDcDdwZzu8EvgzcHFd8w5GsTjCxtlqN5USkrMV5BjEXaHL3re7eBTwMLCwosxB4MJxeBZxvZubu7e7+DEGiGJca69WaWkTKW5wJYhqwPe91czgvsoy79wB7gUypOzCzJWa2wcw27Ny5c5jhDk5DOkmrRpUTkTIWZ4KwiHmFTY9LKVOUu9/n7nPcfc7UqVMHFdxwZdRhn4iUuTgTRDNwXN7r6cCOYmXMrBqYBLTGGNOIydQnVcUkImUtzgTxPHCimc00sySwCFhTUGYNcF04fQXwpL9HOjjKpFO0tneRy70nwhURGbTquDbs7j1mdhOwDqgCVrj7ZjNbBmxw9zXA/cBKM2siOHNY1Lu+mW0DjgKSZnYJcKG7vxJXvIPVkE6SzTn7OruZXJcc63BEREZcbAkCwN3XAmsL5t2WN90JXFlk3RlxxjZc+a2plSBEpBypJfUQqTW1iJQ7JYghUmtqESl3ShBD1BhWMelOJhEpV0oQQzRFXX6LSJlTghiimqoEkybUqIpJRMqWEsQwZNJqLCci5UsJYhgy9epuQ0TKlxLEMAQd9ilBiEh5UoIYhkx9ihb16CoiZUoJYhgy4RmE+mMSkXKkBDEMmXSSnMOeA91jHYqIyIhTghiGhvre7jZUzSQi5UcJYhga0/0d9omIlBsliGFoqFdrahEpX0oQw9Dbo6taU4tIOVKCGIYpdTWAOuwTkfKkBDEM1VUJptTVqDW1iJQlJYhhUmtqESlXShDDlKlPsUvXIESkDClBDFNGZxAiUqaUINp3wXcugBcegq6OQa/eoC6/RaRMKUHsexO62mDNX8JdH4J/uxV2NZW8eqY+xe6OLrLqj0lEyowSxDGnw5//Gj73b3DCBfDct+GbZ8JDC+HVf4Fsz4CrZ9JJ3GFPh84iRKS8VI91AOOCGbz/nOCx/3/Biw/Bhn+GR66BicfCnM/BGdfCxN87bNVM2Jq6pb2LTNg3k4hIOdAZRKGJ74PzboEvvASL/g8c/SF46m/h7lPgh4th2zPg/dVJDWF/TD975R22t3bgrqomESkPVi4/aHPmzPENGzbEs/GWN2DDCnhxJXTuhakfgrNugNOu5t2uJH981y/Z1xlURTXWp5h13GRmHz+Z2cdN5rTjJlOf0omaiIxPZrbR3edELlOCGISuDtj8o+A6xVuboCYNp19N9+zFvJ47jhff3M+L/283m7bvYevOdiCovTrp6Il9SWPW8ZM58eiJVCUs3lhFREqgBBGHNzfC8/fDb1dB9iAkqmHiMXDUNJg0jYN1x7A9O4VX2yeycU8dv3w7xbbOCTgJ0skqTpseJozjgsfUiSnMlDREZHQpQcSpozW422n3tuCW2b1vBs/7dgSJI48nknTUHs3ORCP/2TWZ1w8cxZu5Bt72BtosTSI1keoJR5Gsm0gyPYn69FFMrk8ypS5JQ12SyXU1NKSTTK5LMqWuhsl1SZ2JiMiwDJQgYq0cN7P5wNeBKuA77n5HwfIU8BBwJtACXO3u28JltwLXA1ngv7n7ujhjHbK6BjjzusPnuweN8PY1B8li75vYvmbSe98kvW8HM/Zt5Q973sJyecOV5oD28LETsiTo8BRtTKDda2mjlnafwFZqaWMCHaTorkqTq0nj1ROgKgnVKRLVKahJYdW1VNWkqEpOIFGTojpZS3WylprkBGpStdTU1pFM1ZJKTaC6JklNdTXJ6gQ1VQlqqix87p9OViVIKCGJVIzYEoSZVQHLgQuAZuB5M1vj7q/kFbse2O3uJ5jZIuBO4GozOxlYBJwCHAv83MxOcvdsXPGOODOonxo8jp0dXSSXg/Z3gwRycB8cbAsa7R3cD11tVHW1U39wP7UH9jPpwD6yB/aTO7gfDraR6H6bqu52arId1HR3wQgMi511o4eqQx7dVNFBNVlP9M3LWjVZqshZOG3VuCVwqnBLkLMq3ILp/GcsgVt1/3SiCvrmV0EiAdb7qOqbtkSw3BL5ywyzKkgE880sb34CEongufe15b1OVGGJcP1webB+/jrh/IQBFu4jcUj5/u2F69O7nvXHZ4YZJCyB926XBCSC6UQi0bcelsDCz47lP8LyQTnC/ZMXs2EYWLguQdzBtsIYw4/k4dvK+/vpf907TX48WHg8eneVP82h60hZiPMMYi7Q5O5bAczsYWAhkJ8gFgJfCadXAd+04BO2EHjY3Q8C/2lmTeH2fhNjvKMvkQjaVkS0r+hlQE34KCrbDT2d0NMVVGsdMt0FPZ14z0G6uzrp6uygu6uTnq5Oeg4eoKe7k2zXQTzbTS7bhWezkO0il+2BXHew7Vw2fO7Bcj3U5LqpyWWxXDcJD+aZd2OewzyLkcM8RyKXJUE2mPYcRpYqz5IgFzw8fCZLAsfIkcBJUB7VnuUu50EycCBH73T/vP7XUfMKyw60fv8zhfPt0PLkbav4PvpLRG0/aluDWZ4fVyGPiK0w5mL7iN5P8PzW1Hmc8+f3Ru5zOOJMENOA7Xmvm4Gzi5Vx9x4z2wtkwvnPFqw7rXAHZrYEWAJw/PHHj1jg7zlVNcFjgHZ6BiTDx7jnDp7rf+Syea+z4E4umyWb6yGbzeK5LJ5zcrksOXfIZcl5jlzOg2Wew3NZcg6e7X2dIxc+B2UI5ofzgHC+456D8DmYpr9c4Xx6Y/dgOtc/7cFOCDcQvs6FbWfyl4GTC4+Dh81ucnnHxg9bJygTTJt78ANzSBn61rG86b4yfeX73wPHsTCe8IDkLe7/CT98/WAdoz82C45G8NwXK4fs3w67Hprr29whE4fs+9C4LZzXt69wu963PHq9/L8778Uh2yy2PHI7hWVK2U5vOT/0b+ifLtx0//thkw77eRwRcSaIqBRaeCSLlSllXdz9PuA+CC5SDzZAGad6q4uoKlokET4GPLMSkWGJsyV1M3Bc3uvpwI5iZcysGpgEtJa4roiIxCjOBPE8cKKZzTSzJMFF5zUFZdYAvbcAXQE86cG56xpgkZmlzGwmcCLwXIyxiohIgdiqmMJrCjcB6wjqCla4+2YzWwZscPc1wP3AyvAidCtBEiEs9wOCC9o9wOffU3cwiYiUATWUExGpYAM1lFNvriIiEkkJQkREIilBiIhIJCUIERGJVDYXqc1sJ/C7YWyiEdg1QuGMJMU1OIprcBTX4JRjXO9396lRC8omQQyXmW0odiV/LCmuwVFcg6O4BqfS4lIVk4iIRFKCEBGRSEoQ/e4b6wCKUFyDo7gGR3ENTkXFpWsQIiISSWcQIiISSQlCREQiVVSCMLP5Zva6mTWZ2dKI5SkzeyRcvt7MZoxCTMeZ2VNm9qqZbTazL0SU+UMz22tmm8LHbXHHlbfvbWb223C/h/WGaIFvhMfs383sjJjj+WDecdhkZvvM7IsFZUbteJnZCjN718xezpvXYGY/M7Mt4fOUIuteF5bZYmbXRZUZ4bj+3sxeC9+n1WY2uci6A77nMcT1FTN7M+/9uqjIugN+f2OI65G8mLaZ2aYi68Z5vCJ/H0btM+bh8IXl/iDocvwN4AMEI2++BJxcUOYvgHvD6UXAI6MQ1zHAGeH0ROA/IuL6Q+Bfx+i4bQMaB1h+EfBTglEAPwqsH+X39G2Chj5jcryA84AzgJfz5v0dsDScXgrcGbFeA7A1fJ4STtnAJd0AAAVpSURBVE+JOa4Lgepw+s6ouEp5z2OI6yvAzSW81wN+f0c6roLl/wjcNgbHK/L3YbQ+Y5V0BjEXaHL3re7eBTwMLCwosxB4MJxeBZxvVmT08RHi7m+5+wvh9H7gVSLG3x7HFgIPeeBZYLKZHTNK+z4feMPdh9OCfljc/WmCsUzy5X+OHgQuiVj1E8DP3L3V3XcDPwPmxxmXuz/u7j3hy2cJRmocVUWOVylK+f7GElf4G3AV8P2R2l+pBvh9GJXPWCUliGnA9rzXzRz+Q9xXJvwi7QUyoxIdEFZpzQbWRyw+x8xeMrOfmtkpoxUTwVjgj5vZRjNbErG8lOMal0UU/9KO1fECeJ+7vwXBFxw4OqLMWB43gD8jOPOLcqT3PA43hVVfK4pUl4zl8ZoHvOPuW4osH5XjVfD7MCqfsUpKEFFnAoX3+JZSJhZmVg88CnzR3fcVLH6BoBrldOB/A4+NRkyhc939DGAB8HkzO69g+ZgcMwuGsf0U8MOIxWN5vEo1lp+1vyIYqfF7RYoc6T0fafcAvw/MAt4iqM4pNGbHC/g0A589xH68jvD7UHS1iHmDOmaVlCCagePyXk8HdhQrY2bVwCSGdjo8KGZWQ/Dmf8/df1S43N33uXtbOL0WqDGzxrjjCve3I3x+F1hNcKqfr5TjGocFwAvu/k7hgrE8XqF3eqvZwud3I8qMyXELL1ReDPyphxXVhUp4z0eUu7/j7ll3zwHfLrK/sTpe1cBlwCPFysR9vIr8PozKZ6ySEsTzwIlmNjP873MRsKagzBqg90r/FcCTxb5EIyWs37wfeNXd7ypS5vd6r4WY2VyC960lzrjCfaXNbGLvNMFFzpcLiq0BrrXAR4G9vae+MSv6X91YHa88+Z+j64AfR5RZB1xoZlPCKpULw3mxMbP5wJeAT7l7R5EypbznIx1X/jWrS4vsr5Tvbxz+GHjN3ZujFsZ9vAb4fRidz1gcV97H64Pgjpv/ILgb4q/CecsIvjAAtQRVFk3Ac8AHRiGmjxGc9v07sCl8XATcCNwYlrkJ2Exw58azwB+M0vH6QLjPl8L99x6z/NgMWB4e098Cc0YhrjqCH/xJefPG5HgRJKm3gG6C/9iuJ7hu9QSwJXxuCMvOAb6Tt+6fhZ+1JuBzoxBXE0GddO/nrPeOvWOBtQO95zHHtTL87Pw7wQ/fMYVxha8P+/7GGVc4/597P1d5ZUfzeBX7fRiVz5i62hARkUiVVMUkIiKDoAQhIiKRlCBERCSSEoSIiERSghARkUhKECLjgAU90P7rWMchkk8JQkREIilBiAyCmV1jZs+Fff9/y8yqzKzNzP7RzF4wsyfMbGpYdpaZPWv94y9MCeefYGY/DzsTfMHMfj/cfL2ZrbJgzIbvxd2TsMiRKEGIlMjMPgxcTdA52ywgC/wpkCboF+oM4JfA34SrPAR8yd1PI2gp3Dv/e8ByDzoT/AOCFrwQ9NT5RYL+/j8AnBv7HyUygOqxDkDkPeR84Ezg+fCf+wkEnaTl6O/M7bvAj8xsEjDZ3X8Zzn8Q+GHYb880d18N4O6dAOH2nvOwzx8LRi+bATwT/58lEk0JQqR0Bjzo7rceMtPsywXlBuq/ZqBqo4N501n0/ZQxpiomkdI9AVxhZkdD37jA7yf4Hl0RlvkM8Iy77wV2m9m8cP5ngV960Jd/s5ldEm4jZWZ1o/pXiJRI/6GIlMjdXzGzvyYYPSxB0PPn54F24BQz20gwCuHV4SrXAfeGCWAr8Llw/meBb5nZsnAbV47inyFSMvXmKjJMZtbm7vVjHYfISFMVk4iIRNIZhIiIRNIZhIiIRFKCEBGRSEoQIiISSQlCREQiKUGIiEik/w9WDfvFS7xa6wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "import random\n",
    "from IPython import display\n",
    "from matplotlib import pyplot as plt\n",
    "from torch import nn\n",
    "import torch.utils.data as Data\n",
    "import torch.optim as optim\n",
    "from torch.nn import init\n",
    "\n",
    "num_inputs = 200\n",
    "#1类\n",
    "x1 = torch.normal(2,1,(10000, num_inputs))\n",
    "y1 = torch.ones(10000,1) # 标签1 \n",
    "x1_train = x1[:7000]\n",
    "x1_test = x1[7000:]\n",
    "#0类\n",
    "x2 = torch.normal(-2,1,(10000, num_inputs))\n",
    "y2 = torch.zeros(10000,1) # 标签0\n",
    "x2_train = x2[:7000]\n",
    "x2_test = x2[7000:]\n",
    "#合并训练集\n",
    "trainfeatures = torch.cat((x1_train,x2_train), 0).type(torch.FloatTensor)\n",
    "trainlabels = torch.cat((y1[:7000], y2[:7000]), 0).type(torch.FloatTensor)\n",
    "#合并测试集\n",
    "testfeatures = torch.cat((x1_test,x2_test), 0).type(torch.FloatTensor)\n",
    "testlabels = torch.cat((y1[7000:], y2[7000:]), 0).type(torch.FloatTensor)\n",
    "print(trainfeatures.shape,trainlabels.shape,testfeatures.shape,testlabels.shape)\n",
    "\n",
    "batch_size = 50\n",
    "# 将训练数据的特征和标签组合\n",
    "dataset = Data.TensorDataset(trainfeatures, trainlabels)\n",
    "# 把 dataset 放入 DataLoader\n",
    "train_iter = Data.DataLoader(\n",
    "    dataset=dataset, # torch TensorDataset format\n",
    "    batch_size=batch_size, # mini batch size\n",
    "    shuffle=True, # 是否打乱数据 (训练集一般需要进行打乱)\n",
    "    num_workers=0, # 多线程来读数据， 注意在Windows下需要设置为0\n",
    ")\n",
    "# 将测试数据的特征和标签组合\n",
    "dataset = Data.TensorDataset(testfeatures, testlabels)\n",
    "# 把 dataset 放入 DataLoader\n",
    "test_iter = Data.DataLoader(\n",
    "    dataset=dataset, # torch TensorDataset format\n",
    "    batch_size=batch_size, # mini batch size\n",
    "    shuffle=False, # 是否打乱数据 (训练集一般需要进行打乱)\n",
    "    num_workers=0, # 多线程来读数据， 注意在Windows下需要设置为0\n",
    ")\n",
    "#实现FlattenLayer层\n",
    "#完成将数据集展平的操作，保证一个样本的数据变成一个数组\n",
    "class FlattenLayer(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "         super(FlattenLayer, self).__init__()\n",
    "    def forward(self, x):\n",
    "        return x.view(x.shape[0],-1)\n",
    "# #模型构建\n",
    "num_hiddens,num_outputs = 256,1\n",
    "net = nn.Sequential(\n",
    "        FlattenLayer(),\n",
    "        nn.Linear(num_inputs,num_hiddens),\n",
    "        nn.ReLU(),\n",
    "        nn.Linear(num_hiddens,num_outputs),\n",
    "        )\n",
    "for params in net.parameters():\n",
    "    init.normal_(params,mean=0,std=0.01)\n",
    "# 定义二分类交叉熵损失函数\n",
    "loss = torch.nn.BCEWithLogitsLoss()\n",
    "# 定义sgd优化器\n",
    "optimizer = torch.optim.SGD(net.parameters(),lr)\n",
    "#定义模型训练函数\n",
    "def train2(net,train_iter,test_iter,loss,num_epochs,batch_size,params=None,lr=None,optimizer=None):\n",
    "    train_ls = []\n",
    "    test_ls = []\n",
    "    for epoch in range(num_epochs): # 训练模型一共需要num_epochs个迭代周期\n",
    "        train_l_sum, train_acc_num,n = 0.0,0.0,0\n",
    "        # 在每一个迭代周期中，会使用训练数据集中所有样本一次\n",
    "        for X, y in train_iter: # x和y分别是小批量样本的特征和标签\n",
    "            y_hat = net(X)\n",
    "            l = loss(y_hat, y.view(-1,1)) # l是有关小批量X和y的损失\n",
    "            #梯度清零\n",
    "            if optimizer is not None:\n",
    "                optimizer.zero_grad()\n",
    "            elif params is not None and params[0].grad is not None:\n",
    "                for param in params:\n",
    "                    param.grad.data.zero_()\n",
    "            l.backward() # 小批量的损失对模型参数求梯度\n",
    "            if optimizer is None:\n",
    "                SGD(params,lr)\n",
    "            else:\n",
    "                optimizer.step()\n",
    "            #计算每个epoch的loss\n",
    "            train_l_sum += l.item()*y.shape[0]\n",
    "            #每一个epoch的所有样本数\n",
    "            n+= y.shape[0]\n",
    "        train_labels = trainlabels.view(-1,1)\n",
    "        test_labels = testlabels.view(-1,1)\n",
    "        train_ls.append(train_l_sum/n)\n",
    "        test_ls.append(loss(net(testfeatures),test_labels).item())\n",
    "        print(\"epoch %d,train_loss %.6f,test_loss %.6f\"%(epoch+1,train_loss[epoch],test_loss[epoch])) \n",
    "    return train_ls,test_ls\n",
    "lr = 0.01\n",
    "num_epochs = 20\n",
    "train_loss,test_loss = train2(net,train_iter,test_iter,loss,num_epochs,batch_size,net.parameters,lr,optimizer)\n",
    "import matplotlib.pyplot as plt\n",
    "x=np.linspace(0,len(train_loss),len(train_loss))\n",
    "plt.plot(x,train_loss,label=\"train_loss\",linewidth=1.5)\n",
    "plt.plot(x,test_loss,label=\"test_loss\",linewidth=1.5)\n",
    "plt.xlabel(\"epoch\")\n",
    "plt.ylabel(\"loss\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.1428, -2.8546, -1.6359,  0.6920, -0.6296, -0.5586, -4.8998,  6.9726,\n",
      "         -0.7776,  3.1059],\n",
      "        [ 2.3096, -0.4873,  4.6295,  2.2892, -4.0116,  2.1053,  3.7338, -6.5171,\n",
      "          1.7106, -5.4087],\n",
      "        [-3.0559,  4.7891,  0.8791,  0.6430, -1.6265, -0.7474, -0.3839, -0.2103,\n",
      "          0.8146, -0.7833],\n",
      "        [ 7.2414, -6.7511,  0.1606, -0.1565, -2.6850,  2.9878,  1.2709, -0.3333,\n",
      "         -0.5054, -1.5477],\n",
      "        [-0.8085, -4.2709,  0.3944, -2.2240,  3.9578, -0.9954,  0.3904,  0.6743,\n",
      "          0.3383,  2.4993],\n",
      "        [-4.1172,  5.6425,  0.6610,  1.2124, -1.9018, -1.0414, -1.8777,  0.6324,\n",
      "          1.3275, -0.2229],\n",
      "        [-2.8439, -2.8956, -2.9791, -0.1491,  3.7473,  1.3317, -1.8261,  0.8564,\n",
      "          1.4315,  3.2325],\n",
      "        [-3.8977, -0.8409, -1.9047, -0.4439,  2.6862,  0.4297, -0.2862,  0.1360,\n",
      "          1.0550,  3.1776],\n",
      "        [ 0.7926, -2.5289,  2.0524, -3.1436,  1.7284,  0.4656,  3.8618, -3.4004,\n",
      "          0.9344, -0.6036],\n",
      "        [-1.5528, -3.7579, -3.9060, -2.5369,  3.4794,  0.4309, -2.3854,  4.2482,\n",
      "          0.4861,  5.2169],\n",
      "        [ 6.7918, -4.6346,  1.9025,  2.1138, -3.6824,  3.9318, -0.3693, -4.0090,\n",
      "          1.9958, -4.2369],\n",
      "        [ 0.6557, -1.3480,  2.0828, -0.3701,  0.2810,  0.4569,  2.7128, -3.6833,\n",
      "          1.2450, -1.8229],\n",
      "        [-1.7841, -4.2198, -2.8282, -2.2155,  3.8661,  0.1596, -1.6297,  2.7908,\n",
      "          0.6095,  5.0448],\n",
      "        [ 6.3192, -6.4351, -0.4408, -0.4726, -1.7153,  3.2345, -1.3015, -1.0314,\n",
      "          1.8512, -0.3534],\n",
      "        [-4.0543,  5.9775,  0.6886,  1.8343, -2.7918, -0.3957, -0.5204, -0.5560,\n",
      "          1.2057, -1.0085],\n",
      "        [ 1.2875, -1.8231,  0.2316,  2.5555, -1.9087,  2.7950, -1.2167, -2.1221,\n",
      "          1.9886, -1.7618],\n",
      "        [-1.2363, -4.5313, -0.4348, -1.8008,  3.2738, -1.2454, -1.2778,  2.5148,\n",
      "          0.5145,  4.0724],\n",
      "        [ 1.4343, -3.3690, -1.2224,  2.0500, -1.8334, -0.1427, -4.6239,  6.6453,\n",
      "         -1.3867,  2.0087],\n",
      "        [-1.5094, -0.9400,  0.8214,  2.2416, -0.7001,  1.5390,  1.4028, -2.7614,\n",
      "          1.2797, -1.1270],\n",
      "        [-2.2274, -3.6475, -1.8471, -1.2865,  4.9717,  0.3147,  0.0572, -0.0403,\n",
      "          0.2801,  3.3836],\n",
      "        [-1.9106, -2.5794, -4.7791,  0.3911,  1.7881,  1.4058, -4.8271,  4.7256,\n",
      "          0.6692,  4.8001],\n",
      "        [-0.3911, -3.4046,  0.7769, -0.9072,  2.2207,  2.0064,  5.5884, -5.8229,\n",
      "          1.2359, -0.9202],\n",
      "        [-2.2935,  0.3222,  1.9162, -2.0994,  2.2727, -2.2066,  3.3155, -0.8076,\n",
      "         -0.3012,  0.1619],\n",
      "        [ 1.2796, -3.2415, -1.4247,  0.5997, -0.0095,  3.9155,  0.8688, -3.4161,\n",
      "          1.9814, -0.4696],\n",
      "        [-1.9539, -2.6939, -0.5461, -0.8185,  3.5266, -0.7449, -0.5108,  1.2848,\n",
      "         -0.3675,  2.8013],\n",
      "        [ 8.9505, -7.9072,  1.0188, -1.7991, -1.6798,  4.1983,  2.7694, -3.3385,\n",
      "          0.6985, -3.2166],\n",
      "        [-0.2239, -2.7404, -1.8953,  0.4221,  0.3304, -0.4127, -3.4507,  5.4249,\n",
      "         -1.0716,  3.3069],\n",
      "        [-1.3150, -5.0571, -1.4038, -2.4117,  5.4010,  0.4737,  0.5380, -0.5679,\n",
      "          0.8692,  3.4535],\n",
      "        [ 6.7984, -6.1463,  0.7963,  2.1928, -3.8414,  3.5388, -1.6388, -1.5477,\n",
      "          1.6334, -2.0835],\n",
      "        [-3.1080,  3.8843, -0.0475,  0.8913, -1.3997,  0.6586,  0.3060, -1.2585,\n",
      "          1.3402, -0.9378],\n",
      "        [-0.9253, -0.5695, -2.0313,  5.3614, -3.1773,  2.6811, -3.3618,  1.1097,\n",
      "          0.5799,  0.2723],\n",
      "        [-3.2015,  3.3518, -0.3284,  1.3049, -1.1168,  0.0474, -0.7719,  0.0670,\n",
      "          0.7527,  0.0934]], grad_fn=<AddmmBackward>)\n",
      "epoch 1,train_loss 1.071230,test_loss 0.474177,train_acc 0.763850,test_acc 0.880200\n",
      "epoch 2,train_loss 0.416138,test_loss 0.352211,train_acc 0.886717,test_acc 0.901000\n",
      "epoch 3,train_loss 0.347587,test_loss 0.315264,train_acc 0.901833,test_acc 0.910900\n",
      "epoch 4,train_loss 0.318684,test_loss 0.296245,train_acc 0.909100,test_acc 0.915600\n",
      "epoch 5,train_loss 0.300602,test_loss 0.283022,train_acc 0.914567,test_acc 0.919900\n",
      "epoch 6,train_loss 0.287170,test_loss 0.271513,train_acc 0.917217,test_acc 0.922500\n",
      "epoch 7,train_loss 0.275747,test_loss 0.262670,train_acc 0.921333,test_acc 0.925600\n",
      "epoch 8,train_loss 0.265544,test_loss 0.252448,train_acc 0.924033,test_acc 0.927900\n",
      "epoch 9,train_loss 0.255264,test_loss 0.244514,train_acc 0.926633,test_acc 0.930000\n",
      "epoch 10,train_loss 0.245727,test_loss 0.236345,train_acc 0.929700,test_acc 0.932700\n",
      "epoch 11,train_loss 0.236009,test_loss 0.228369,train_acc 0.932883,test_acc 0.934200\n",
      "epoch 12,train_loss 0.226716,test_loss 0.219387,train_acc 0.935483,test_acc 0.936400\n",
      "epoch 13,train_loss 0.217517,test_loss 0.210440,train_acc 0.938300,test_acc 0.939100\n",
      "epoch 14,train_loss 0.208830,test_loss 0.204421,train_acc 0.940533,test_acc 0.940900\n",
      "epoch 15,train_loss 0.200674,test_loss 0.196581,train_acc 0.943333,test_acc 0.941800\n",
      "epoch 16,train_loss 0.193062,test_loss 0.189691,train_acc 0.945317,test_acc 0.944600\n",
      "epoch 17,train_loss 0.185665,test_loss 0.182633,train_acc 0.947917,test_acc 0.946700\n",
      "epoch 18,train_loss 0.179104,test_loss 0.176789,train_acc 0.949133,test_acc 0.949000\n",
      "epoch 19,train_loss 0.172524,test_loss 0.170999,train_acc 0.951167,test_acc 0.950300\n",
      "epoch 20,train_loss 0.166483,test_loss 0.167396,train_acc 0.953333,test_acc 0.951500\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deZycVZ3v8c+v16reqnpLb0lIAoRFloAhAQHBQTEJCCoawEFARa73issoKFwdR7lzx3XUcS6C6MAoOoCAICpMUIZFB5KQYICsZE93Op3udKf3vfvcP56nuyud6iWhq6vTz/f9etWrluc8Vaerq+vb5znPOcecc4iISHClJLsCIiKSXAoCEZGAUxCIiAScgkBEJOAUBCIiAZeW7AocqaKiIjdnzpxkV0NE5Jiydu3aA8654njbjrkgmDNnDmvWrEl2NUREjilmtnukbTo0JCIScAoCEZGAUxCIiATcMddHICLTT09PD1VVVXR2dia7Kse8UCjEzJkzSU9PH/c+CgIRSbqqqipyc3OZM2cOZpbs6hyznHPU19dTVVXF3Llzx72fDg2JSNJ1dnZSWFioEHiLzIzCwsIjblkpCERkSlAITIyjeR8DEwSv7Grg2/+5GU27LSJyqMAEwetVTdz9/HYOtvckuyoiIlNKYIKgPBICoLqxI8k1EZGpprGxkR//+MdHvN+yZctobGw84v1uvPFGHn300SPeL1ECEwRl0TAANU06PU1EDjVSEPT19Y2631NPPUU0Gk1UtSZNYE4fLfNbBPua1CIQmcq+8bsNbKxuntDnPLU8j39439tG3H777bezfft2FixYQHp6Ojk5OZSVlbFu3To2btzI+9//fiorK+ns7ORzn/scN998MzA091lraytLly7lggsu4KWXXqKiooLf/va3hMPhMev27LPPcuutt9Lb28s555zD3XffTWZmJrfffjtPPvkkaWlpXHrppXzve9/jkUce4Rvf+AapqalEIhFefPHFCXl/AhMERTmZpKUY1WoRiMgw3/rWt1i/fj3r1q3j+eef57LLLmP9+vWD5+Lfd999FBQU0NHRwTnnnMNVV11FYWHhIc+xdetWHnzwQX7605+yfPlyHnvsMa677rpRX7ezs5Mbb7yRZ599lvnz53P99ddz9913c/311/P444+zefNmzGzw8NOdd97JihUrqKioOKpDUiMJTBCkphgleSEdGhKZ4kb7z32yLFq06JABWT/60Y94/PHHAaisrGTr1q2HBcHcuXNZsGABAG9/+9vZtWvXmK+zZcsW5s6dy/z58wG44YYbuOuuu7jlllsIhULcdNNNXHbZZVx++eUAnH/++dx4440sX76cD37wgxPxowIB6iMA7/CQOotFZCzZ2dmDt59//nn+9Kc/8fLLL/Paa69x1llnxR2wlZmZOXg7NTWV3t7eMV9npNPZ09LSWL16NVdddRVPPPEES5YsAeCee+7hH//xH6msrGTBggXU19cf6Y8W//Um5FmOEWXRMK9XTVxzSkSmh9zcXFpaWuJua2pqIj8/n6ysLDZv3szKlSsn7HVPPvlkdu3axbZt2zjhhBN44IEHuOiii2htbaW9vZ1ly5Zx7rnncsIJJwCwfft2Fi9ezOLFi/nd735HZWXlYS2ToxGoICiPhFixoRPnnEYxisigwsJCzj//fE477TTC4TAlJSWD25YsWcI999zDGWecwUknncS55547Ya8bCoW4//77+fCHPzzYWfypT32KhoYGrrzySjo7ve+rH/zgBwDcdtttbN26Feccl1xyCWeeeeaE1MOOtZG2CxcudEe7Qtn9/72Tb/xuI2u++m6KcjLH3kFEJsWmTZs45ZRTkl2NaSPe+2lma51zC+OVD1gfgcYSiIgMF6ggKI9qdLGITJ5Pf/rTLFiw4JDL/fffn+xqHSZQfQSlg4PK1CIQkcS76667kl2FcUlYi8DM7jOzWjNbP8J2M7Mfmdk2M3vdzM5OVF0GFGVnkp5qCgIRkRiJPDT078CSUbYvBU70LzcDdyewLgCkpBilkZCmmRARiZGwIHDOvQg0jFLkSuAXzrMSiJpZWaLqM6AsL8y+RrUIREQGJLOzuAKojLlf5T92GDO72czWmNmaurq6t/SiZdEQ+5rVIhARGZDMIIg3oivuoAbn3L3OuYXOuYXFxcVv6UXLImFqmjrp7z+2xk+ISOIc7XoEAD/84Q9pb28ftcycOXM4cODAUT3/ZEhmEFQBs2LuzwSqE/2iZZEQPX2OA21diX4pETlGJDoIprpknj76JHCLmT0ELAaanHP7Ev2iA+sS1DR1MiM3lOiXE5Ej9fTtUPPGxD5n6emw9Fsjbo5dj+A973kPM2bM4Ne//jVdXV184AMf4Bvf+AZtbW0sX76cqqoq+vr6+Pu//3v2799PdXU173rXuygqKuK5554bsyrf//73ue+++wC46aab+PznPx/3ua+++uq4axIkQsKCwMweBC4GisysCvgHIB3AOXcP8BSwDNgGtAMfS1RdYg2MLq5u7OSMmZPxiiIy1cWuR/DMM8/w6KOPsnr1apxzXHHFFbz44ovU1dVRXl7OH/7wB8CbjC4SifD973+f5557jqKiojFfZ+3atdx///2sWrUK5xyLFy/moosuYseOHYc9d0NDQ9w1CRIhYUHgnLt2jO0O+HSiXn8kZVGtVCYypY3yn/tkeOaZZ3jmmWc466yzAGhtbWXr1q1ceOGF3HrrrXz5y1/m8ssv58ILLzzi5/7LX/7CBz7wgcFprj/4wQ/y5z//mSVLlhz23L29vXHXJEiEQE0xAVCYnUFGaormGxKRuJxz3HHHHaxbt45169axbds2PvGJTzB//nzWrl3L6aefzh133MGdd955VM8dT7znHmlNgkQIXBCYeYPKtGSliAyIXY/gve99L/fddx+tra0A7N27l9raWqqrq8nKyuK6667j1ltv5dVXXz1s37G8853v5IknnqC9vZ22tjYef/xxLrzwwrjP3draSlNTE8uWLeOHP/wh69atS8wPT8DmGhpQFgmxTxPPiYgvdj2CpUuX8pGPfITzzjsPgJycHH75y1+ybds2brvtNlJSUkhPT+fuu73JEG6++WaWLl1KWVnZmJ3FZ599NjfeeCOLFi0CvM7is846ixUrVhz23C0tLXHXJEiEQK1HMODvHl7H6p0N/PftfzNBtRKRt0LrEUwsrUcwDqWREPubO+nToDIRkWAeGiqPhOjtd9S3djEjT2MJRGRiLF68mK6uQwerPvDAA5x++ulJqtH4BDIIBscSNHUqCESmiOmwlviqVauSXYURz0waTWAPDQHqMBaZIkKhEPX19Uf1JSZDnHPU19cTCh3ZP7iBbBGUR70WgRaoEZkaZs6cSVVVFW91dmHxQnXmzCObNiGQQZCflU5mWopGF4tMEenp6cydOzfZ1QisQB4aMjPKNKhMRAQIaBDA0LoEIiJBF9wgiGp0sYgIBDkIIiH2t3RpUJmIBF6AgyBMX7+jrkUrlYlIsAU2CMr9dQmqdeaQiARcYIOgNM8fS9CoDmMRCbbABkG5VioTEQECHASRcDrh9FSNLhaRwAtsEAwMKlOLQESCLrBBAP5YArUIRCTgAh0EpXlhdRaLSOAFOgjKoyFqWzrp7etPdlVERJIm0EFQFgnT76BWg8pEJMACHgQ6hVREJNhBMDiWQP0EIhJcwQ6CiEYXi4gEOgjyQmlkZaRqviERCbRAB8HAoDItUCMiQRboIABvIXstWSkiQRb4ICjN00plIhJsgQ+CsmiYutYuejSoTEQCKvBBUB4J4Rzsb9bhIREJpsAHQWlEYwlEJNgSGgRmtsTMtpjZNjO7Pc722Wb2nJn91cxeN7NliaxPPOVRfyyBgkBEAiphQWBmqcBdwFLgVOBaMzt1WLGvAr92zp0FXAP8OFH1GcngNBPqMBaRgEpki2ARsM05t8M51w08BFw5rIwD8vzbEaA6gfWJKzeUTk5mmloEIhJYaQl87gqgMuZ+FbB4WJmvA8+Y2WeAbODdCazPiLRSmYgEWSJbBBbnMTfs/rXAvzvnZgLLgAfM7LA6mdnNZrbGzNbU1dVNeEXLomG1CEQksBIZBFXArJj7Mzn80M8ngF8DOOdeBkJA0fAncs7d65xb6JxbWFxcPOEVLcsLUa2J50QkoBIZBK8AJ5rZXDPLwOsMfnJYmT3AJQBmdgpeEEz8v/xjKIuGONDaRXevBpWJSPAkLAicc73ALcAKYBPe2UEbzOxOM7vCL/ZF4JNm9hrwIHCjc2744aOEGzhzSIPKRCSIEtlZjHPuKeCpYY99Leb2RuD8RNZhPAbWJahu7GBWQVaSayMiMrkCP7IYvEXsAWrUIhCRAFIQAKWDLQIFgYgEj4IAyMlMIzeURo3GEohIACkIfOURLVAjIsGkIPCVanSxiASUgsBXHtXaxSISTAoCX1kkzIHWbrp6+5JdFRGRSaUg8A0sUKNWgYgEjYLAVx7RAjUiEkwKAl9ZdGDJSnUYi0iwKAh8A/MNaVCZiASNgsCXlZFGJJyuPgIRCRwFQQytVCYiQaQgiFEW0QI1IhI8CoIYZdGwZiAVkcBREMQoj4RoaOums0eDykQkOBQEMUo1lkBEAkhBEKM8orEEIhI8CoIYA9NM7FOHsYgEiIIgxsDaxeowFpEgURDECGekkp+VTnWjDg2JSHAoCIYpjYTVWSwigaIgGKY8ElIQiEigKAiGKYtqmgkRCRYFwTBlkTCN7T10dGtQmYgEg4JgmDKNJRCRgFEQDFOm0cUiEjAKgmGGFqhRi0BEgmFcQWBmnzOzPPP8m5m9amaXJrpyyaBF7EUkaMbbIvi4c64ZuBQoBj4GfCthtUqiUHoqhdkZVCsIRCQgxhsE5l8vA+53zr0W89i0U6qVykQkQMYbBGvN7Bm8IFhhZrlAf+KqlVxlkbAODYlIYKSNs9wngAXADudcu5kV4B0empbKoyFW76xPdjVERCbFeFsE5wFbnHONZnYd8FWgKXHVSq7SSIjmzl7aunqTXRURkYQbbxDcDbSb2ZnAl4DdwC8SVqskK9dYAhEJkPEGQa9zzgFXAv/inPsXIHesncxsiZltMbNtZnb7CGWWm9lGM9tgZv8x/qonjkYXi0iQjLePoMXM7gA+ClxoZqlA+mg7+GXuAt4DVAGvmNmTzrmNMWVOBO4AznfOHTSzGUfzQ0y0wdHFWqlMRAJgvC2Cq4EuvPEENUAF8N0x9lkEbHPO7XDOdQMP4bUoYn0SuMs5dxDAOVc77ponUEkkE9ChIREJhnEFgf/l/ysgYmaXA53OubH6CCqAypj7Vf5jseYD883sv81spZktifdEZnazma0xszV1dXXjqfJbkpmWSlFOhg4NiUggjHeKieXAauDDwHJglZl9aKzd4jzmht1PA04ELgauBX5mZtHDdnLuXufcQufcwuLi4vFU+S0r00plIhIQ4+0j+ApwzsChGzMrBv4EPDrKPlXArJj7M4HqOGVWOud6gJ1mtgUvGF4ZZ70SpiwSYld9W7KrISKScOPtI0gZdvy+fhz7vgKcaGZzzSwDuAZ4cliZJ4B3AZhZEd6hoh3jrFNClUVC6iwWkUAYb4vgP81sBfCgf/9q4KnRdnDO9ZrZLcAKIBW4zzm3wczuBNY45570t11qZhuBPuA259yUGNJbFg3T0tVLS2cPuaFRT5ASETmmjSsInHO3mdlVwPl4x/7vdc49Po79nmJYYDjnvhZz2wFf8C9TSlnMdNQKAhGZzsbbIsA59xjwWALrMqUMjCWoburkxJIxx86JiByzRg0CM2vh8DN9wGsVOOdcXkJqNQUMtQh0CqmITG+jBoFzLrD/CpfkhTCDanUYi8g0pzWLR5CRlkJRTqYGlYnItKcgGEV5JKRBZSIy7SkIRqHRxSISBAqCUZRGQuxr7MA7y1VEZHpSEIyiPBqirbuPFq1UJiLTmIJgFFqXQESCQEEwioGxBNU6c0hEpjEFwSjKol6LoEYdxiIyjSkIRjEjN5MUg32NahGIyPSlIBhFemoKxbmZVKtFICLTmIJgDGWRsA4Nici0piAYQ3k0pM5iEZnWFARjKM3zWgQaVCYi05WCYAzl0RDt3X00d2hQmYhMTwqCMZRqLIGITHMKgjEMjC5Wh7GITFcKgjGUR9UiEJHpLThB4Bw07Dzi3YpzBgaVqUUgItNTcILgxe/B3edD094j2i0tNYWSPC1QIyLTV3CC4IwPg+uDZ756xLuWRUJaslJEpq3gBEH+HLjg72DDb2DH80e0q1YqE5HpLDhBAHD+57xAeOpL0Ns97t0GWgQaVCYi01GwgiA9DEu/Awe2wKq7x71bWTRMZ08/je09CayciEhyBCsIAOa/F05aBs9/e9wdx1qgRkSms+AFAcCSb/odx18ZV/GBINCgMhGZjoIZBPlz4MIvwobHYftzYxYv91cq07oEIjIdBTMIAN7xWcifC0/dNmbHcVFOJmkpppXKRGRaCm4QpIe8juP6rbDyrlGLpqYYJXkhHRoSkWkpuEEAMP9SOOkyeOE70FQ1atGyiBaoEZHpKdhBAH7HcT+sGL3juDSiaSZEZHpSEOQfBxfeChufgO3/NWKx8qg3uliDykRkulEQALzjM1Awz+847opbZH5JLt29/fzTU5sUBiIyrSQ0CMxsiZltMbNtZnb7KOU+ZGbOzBYmsj4jSg/B0u9C/TZ4OX7H8QfPquCG847jp3/eye2PvUFfv8JARKaHhAWBmaUCdwFLgVOBa83s1DjlcoHPAqsSVZdxOfHdcPLl8OJ3obHysM0pKcbXr3gbn/2bE3h4TSWfefBVunr7klBREZGJlcgWwSJgm3Nuh3OuG3gIuDJOuf8DfAdIfk/skm96C9is+N9xN5sZX7j0JL562Sk89UYNn/zFWtq7tai9iBzbEhkEFUDsv9ZV/mODzOwsYJZz7vejPZGZ3Wxma8xsTV1d3cTXdEB0Nrzzi7DpSdj2pxGL3XThPL5z1Rn8ZWsd1//bapo6NBmdiBy7EhkEFuexwQPrZpYC/AD44lhP5Jy71zm30Dm3sLi4eAKrGMc7Put3HH9pxI5jgOXnzOL/feRsXqtq5Np7V1LXMnJZEZGpLJFBUAXMirk/E6iOuZ8LnAY8b2a7gHOBJ5PWYTwgLdPrOG7YDi/966hFl51exs9uOIedB9pY/pOX2aspKETkGJTIIHgFONHM5ppZBnAN8OTARudck3OuyDk3xzk3B1gJXOGcW5PAOo3Pie+GU97nrXMcp+M41kXzi/nlTYs40NrFh+9+ie11rZNUSRGRiZGwIHDO9QK3ACuATcCvnXMbzOxOM7siUa87Yd77Te96xR1jFn37cQU8fPN5dPf1s/yel1m/tynBlRMRmTgJHUfgnHvKOTffOXe8c+7/+o99zTn3ZJyyF0+J1sCA6Cy46DbY9DvYOnLH8YBTy/N45FPvIJSeyrX3rmT1zoZJqKSIyFunkcWjOe8WKDwBnh55xHGsuUXZPPKp8yjOy+T6+1bx3JbaSaikiMhboyAYTVqmN1V1ww546Ufj2qU8GuaR/3EeJ8zI4ZM/X8PvX68eeycRkSRSEIzlhEvglCvgxX+Gg7vHtUthTib/8clzOXt2Pp958K88uHpPgispInL0FATj8d5/AjN4evSxBbHyQun8/OOLuHh+MXf85g1+8sL2BFdSROToKAjGIzoL3vW/4c3/hB+dDWvuG3N5S4BwRio/+ehC3ndmOd98ejPffGoTrV2akkJEphY71qZUXrhwoVuzJgknFznnrVfw/Deh6hWI+NNRLPhbSE0fdde+fsfXfrueX63aQ3ZGKu87s5xrFs3mzJkRzOINwBYRmVhmttY5F3fAroLgSDkH256F5/8J9q715ye6Dc68dtRAcM7x6p5GHlq9h9+/vo+Onj5OLs3l2kWzef+CCiJZo4eJiMhboSBIBOdg6x+9QKj+K+TP8QLhjGsgNW3UXVs6e/jtumoeemUP6/c2k5mWwrLTy7jmnFksmlugVoKITDgFQSI5B2+u8A4Z7VsH+XPhoi/B6cvHDASA9XubeOiVPfz2r9W0dPUyrziba86ZxVVnz6QwJ3MSfgARCQIFwWRwDrY87QVCzetQcLwXCKd9aFyB0N7dyx9e38dDr1SydvdB0lONS08t5ZpFszj/+CJSUtRKEJGjpyCYTM7B5j/A89+C/W94I5Mv+jKcdhWkpI7rKd7c38JDqyv5zV+raGzvYVZBmKsXzuJDb59FaSSU4B9ARKYjBUEy9PfD5t97gVC7AYrmwwVfgPnvhayCcT1FZ08fKzbU8PArlby0vR6AE2fkcO68QhbPK2Dx3EKKc3X4SETGpiBIpv5+b8WzF74NtRsBg9LTYe47Yd7FMPs8yMwZ82l2HWjjD2/sY9XOBtbuaqCt21sveV5xNovnFnKuHwxqMYhIPAqCqaC/3xt/sPMF2PkiVK6Cvm5ISYOKt8Pci7xwmLXIm+NoFL19/ayvbmbVjnpW7qhnza6DtPgD1eYUZrF4rt9imFdIRTQ8GT+diExxCoKpqLvdC4OdL3rhUP1XcP2QFoLZ53qhMPdiKDtzzM7mvn7HxupmVu2sZ+WOBlbvrKe50wuGmfnhoWCYW8DsgiydnioSQAqCY0FnE+x+CXb4LYbaDd7jmXkw5wIvGOZcAMUnj2sk8+aaZlbtaGDVznpW72zgYHsPALmZaZxclsspZXmcXJrHKWW5nFyaRzhjfB3ZInJsUhAci1rrYNeLXijseAEO7vQeT0n3wqDkbVB6mnddchrkzBjxqfr7HVtrW1mzu4HN+1rYtK+ZzTUtg/MemcHcwmxOKfOCwbvOoywSUutBZJpQEEwHjXtgzyrYvx72b/CuW/YNbc8uHgqFEj8gik8asb+hv99RdbCDjfua2TRwqWmmsqFjsEwknD4UDKVeOMwrziY7c+xxESIytSgIpqu2eu8Q0kAw1KyHus3Q2+ltt1TvtNWStw2FRPF8iMwacUxDS2cPm2tahsJhXwtbalro6OkbLFMeCXH8jBzmFWVz/Iwcji/2LiV5mWpBiExRCoIg6ev1VlSLbTns3wBNlUNl0kLeQLeiE6HoJP96vvdYRtbhT9nv2F3fxuaaFnbUtbK9ro3tda3sqGs7ZFrt7IzUmGDIZp4fEHOKsshMUx+ESDIpCAQ6GqF2Exx4079s9a4bd3tnKw2IzPZaDUXzhwKiaL536GnYf/vOOWpbuthe28r2mIDYXttKdVPnYLkUg1kFWcwryua4wmzmFGZxXFE2xxVkMTM/i4w0LYshkmgKAhlZTyc0bD80HAZu97QPlQtFvRZDdLa3UE9klnc7Msu7n5l7yNO2d/eyYyAYYgJiT0M77d1Dh5lSDCrywxxXkM1xhVnMKcxm9sB1QZbOZhKZIAoCOXL9/dC899CAqN8KTVXepW/YCm2hiNeaGAwJ/3rgtt+icM5xoLWb3fVt7K5vZ3d9G7vq29nd4N1u9E9zHVCSlznUiigcCovjCrPIDWkNB5HxUhDIxOrvh7ZaaKyEpj3+deWh190th+6TFvJCoWAuFMzzpusumOddorMhLQOApvYedjf44XCgbTAgdte3U9ty6HrRhdkZMcGQzZyiLGYXePejWenquBaJoSCQydfROCwc9niXgzuhYSd0tw6VtRSIzBwKhtiQyJ8z2IHd1tXLnoaYVkT9UEhUN3UQ+1HOC6UxJ7ZPwm9FzC7IojgnU9N6S+AoCGRqcQ7a6rxAaNjhXQ7G3O44eGj53LKhgMgr9y8VQ7fD+XT29lN1sJ1dB9rZ5YfDrvo29jS0U3Wwg77+oc95RloKM/PDzC7IYlZ+FrMKvNsz87OYVZBFJKxDTjL9jBYEGhkkk8/MGwmdMwNmLz58e8fBoZAYaEE07IDtz0JLDTDsn5e0EKG8ck7ILeeEgXAoK4eTvNs92fPZ253LroOdVB7soLKh3bscbOfV3QcH52UakBdKY3bhQEj4l/wwswqymJkf1qmwMu0oCGTqCedDRT5UnH34tr4eaN0Pzfu8zuzmamip9q6bq6Fypbetf6jTOR2YY6nMyS3zQiJSAcUVcLzXqmgNlVLVl8/OzmwqG7uobOhgT0M7W/a38OzmWrp7h06vNYPSvBCzCrzDTAOXgftFORnqm5BjjoJAji2p6V5/QmQmcE78Mv390F7vBUVLTGA0V3u3973uLSvqj8DOAU4GTrZU7zBUxD/sVFFBf245TRkl7HP57OzOZ1t7mN0Hu6hsaOfPW+vY33xoB3Y4PXXwUNPwsJiZr9NhZWpSEMj0k5ICOcXehQXxyzjnHYJq3gtNe/2w8AOjqQr2vQZbnialt5N8IB84FbxpOwZaFieU05tTSmNaMfspZHdPlG2deWxszWDXwQ5e2l5/yJgJgKKcTGYVhL3+iHz/2r9fHg3psJMkhYJAgsnMWzI0q8BbMS4e56C9YSgkmqr8Fobfuti/gbStz1DU004R8LbYfbNn4CrK6c4upSl9BgeskKq+fHZ057OxI8obezp4+o1uemM6sc2gJDfEzJj+iFn5WYP3SyMh0lM1ClsmnoJAZCRmkF3oXcrOiF/GOW8tidhDT/7hKGuuJrN5DzOaX2ZGZ5PXohh87lTcjAq6sytoziyjLq2Eqv4itvcUsKk9wqs7cvhtcy8xOUGKQVkkzEy/JeFdD90ui4RIU1DIUVAQiLwVZhCOepeSU0cu193mHYIaGFPRVIk1VpLZuIfiA6sobq7m1EPOhjJcURndORU0Z5ZyILWEKlfEjp4CNnZEWbstl9+0uEPGTqSmGGWR0LCgiGlR5IVI1fgJiUNBIDIZMrK9yfyK58ff3tvtH34aGHxXiTX5QdH4OsXNezml/9DTXF1+Ed05FbSEyjmQOoO9flBs7sjnr2/m8lhr6iFBkZZilOSFqMgPUxENUx4NUR71bnv3w1prIqAS+ls3syXAvwCpwM+cc98atv0LwE1AL1AHfNw5tzuRdRKZktIy/Ok35sbf3t/nHXKKGaltjXvIbKoks3EbRU3PcXJvJ5fE7OLy8ujJmUlL2AuKfRSytyeXXR1ZbN8e4pXWTOr6c+kiY3CfaFY65ZGwHxBeaJRHw4OBoVHZ01PCRhabWSrwJvAeoAp4BQaj4XcAAAwcSURBVLjWObcxpsy7gFXOuXYz+5/Axc65q0d7Xo0sFoljYLR27PxP/iGowdvD53/y9aVl0ZmRT2tqlEbLo64vl5rebPZ0ZlHdm0O9y6PB5VJPHg2WT36et4xpWdTrlxi6ePeLFBZTUrJGFi8CtjnndviVeAi4EhgMAufcczHlVwLXJbA+ItNX7GjtmW8/fLtz0NUMbQe8S/vQdWpbPdntB8huO0BJ+wFOaquErgNgnd5ovGFaeyIcqC9k34F8dvdEqe7LZxP51LgCalwBB6yAcG6hFxTDwqIkL0SpHxY6A2rqSGQQVAAxy2JRBcSZT2DQJ4CnE1gfkeAy86YKD0Wg8PixyzvndXAPBEbbAa/F0VJDTks1Oc3VzGmu5tyW17G2usN27+oO0XCgkJq6fPb0Rqnuz+dlV0Cti1LnIhwgSn/2DPIiUUrzvIAoyQt5tyMhSvIyKc0LEQlrFtnJkMggiPfbi3scysyuAxYCF42w/WbgZoDZs2dPVP1EZCRmkJnjXfLnjFwMoLfLmwNqcBT3PjKbqylrqaaseR8LmndDyyqs/9C1JuiFzoYQDQfz2d8fobovjzoXZbWLUkeEOhelKSUfckrIiMygKJLjB0YmJXkhZuR6t2fkhchRJ/dbksh3rwqYFXN/JlA9vJCZvRv4CnCRc65r+HYA59y9wL3g9RFMfFVF5KilZUL+cd4lDoOhaT9a9/uXWmjdT6i1lvLWGspbazmzpQZat5DS1XToE3R6l8b9edS6CDX9XlC85vIHWxjNaYW4nBJS80rIixRQEgkzI9cLiZLczMEWh6b4iC+RQfAKcKKZzQX2AtcAH4ktYGZnAT8BljjnahNYFxFJpkOm/TgtfpGBGz2d3sJHflgMBEe0pYZI636Ob66hv2UHqe11pMS2Mtq9S0dNJrUuSq2LUOuibHT5vOAi1BGlLTVKf1YRKdmFpOcVk52bT1FuiOKcDIpyMinKzfSuczLIyUwLzGGphAWBc67XzG4BVuCdPnqfc26Dmd0JrHHOPQl8F2/Or0f8N3yPc+6KRNVJRI4B6SF/bezDDwMb3pdJKgzNF9VSExMY+wm37Gd2aw0VTTX0t9SQ0raRtJ6YhZD8Fgb10EMaDS6HBv/MqBpy2ehyaXB5NKfk0RcqoD9cSGpOEem5xYSjxRTk5fhhkUlxrhcgx3pfhhamEZHpr7vdC4r2Bq8DvL3eP2uqnv62enpaaulr9e6ndTaQ0dM84lM1uzAHXS4NeOFxkFwayaUrI5++UD4uq8gPjiKyojPIjRZTlBumKDeDwuxM8rPSkzIViBamEZFgy8gaccBeCpA5/MG+XuhoOCQwaD9Af1s96c115DfXkddWz9z2etI695HZfZD0vi5ow7vEnEjV54xGcjjoctlLmC2E6E7Joi8tm/70bCwzBwvlkhbKJT2cS2Z2hKycCNm5EXIj+eTmRckI53mj0zOyvY78CaYgEBEZLjVtaFxGjBQg7F8O090+GBi01+Pa6ulsqqOjqZbellqyW+vJ6mphRncrKT0NpPVWktHZTqijgwx64j3jYV4/86uc8YHb3upPdxgFgYjIRMjI8i5R72RJY5TQGK6vh77OVlqaD9LUeJCW5kbaWhrpaGumq62JnvYWejubmV2+KCFVVxCIiCRbajqp2flEs/OJlk3+y2uMt4hIwCkIREQCTkEgIhJwCgIRkYBTEIiIBJyCQEQk4BQEIiIBpyAQEQm4Y27SOTOrA452gfsi4MAEVmeiqF5HRvU6clO1bqrXkXkr9TrOOVccb8MxFwRvhZmtGWn2vWRSvY6M6nXkpmrdVK8jk6h66dCQiEjAKQhERAIuaEFwb7IrMALV68ioXkduqtZN9ToyCalXoPoIRETkcEFrEYiIyDAKAhGRgJuWQWBmS8xsi5ltM7Pb42zPNLOH/e2rzGzOJNRplpk9Z2abzGyDmX0uTpmLzazJzNb5l68lul7+6+4yszf811wTZ7uZ2Y/89+t1Mzt7Eup0Usz7sM7Mms3s88PKTNr7ZWb3mVmtma2PeazAzP5oZlv96/wR9r3BL7PVzG5IcJ2+a2ab/d/T42YWHWHfUX/nCarb181sb8zva9kI+47695uAej0cU6ddZrZuhH0T8p6N9N0wqZ8v59y0ugCpwHZgHpABvAacOqzM/wLu8W9fAzw8CfUqA872b+cCb8ap18XA75Pwnu0CikbZvgx4Gm/1vXOBVUn4ndbgDYhJyvsFvBM4G1gf89h3gNv927cD346zXwGww7/O92/nJ7BOlwJp/u1vx6vTeH7nCarb14Fbx/G7HvXvd6LrNWz7PwNfm8z3bKTvhsn8fE3HFsEiYJtzbodzrht4CLhyWJkrgZ/7tx8FLjEzS2SlnHP7nHOv+rdbgE1ARSJfcwJdCfzCeVYCUTObzAX1LgG2O+eOdkT5W+acexFoGPZw7Ofo58D74+z6XuCPzrkG59xB4I/AkkTVyTn3jHOu17+7Epg5Ea91pEZ4v8ZjPH+/CamX/x2wHHhwol5vnHUa6bth0j5f0zEIKoDKmPtVHP6FO1jG/6NpAgonpXaAfyjqLGBVnM3nmdlrZva0mb1tkqrkgGfMbK2Z3Rxn+3je00S6hpH/OJPxfg0occ7tA++PGZgRp0wy37uP47Xk4hnrd54ot/iHre4b4VBHMt+vC4H9zrmtI2xP+Hs27Lth0j5f0zEI4v1nP/wc2fGUSQgzywEeAz7vnGsetvlVvMMfZwL/CjwxGXUCznfOnQ0sBT5tZu8ctj2Z71cGcAXwSJzNyXq/jkRS3jsz+wrQC/xqhCJj/c4T4W7geGABsA/vMMxwSfusAdcyemsgoe/ZGN8NI+4W57Ejfr+mYxBUAbNi7s8EqkcqY2ZpQISja8YeETNLx/tF/8o595vh251zzc65Vv/2U0C6mRUlul7OuWr/uhZ4HK95Hms872miLAVedc7tH74hWe9XjP0Dh8j869o4ZSb9vfM7DC8H/tb5B5KHG8fvfMI55/Y75/qcc/3AT0d4zaR81vzvgQ8CD49UJpHv2QjfDZP2+ZqOQfAKcKKZzfX/m7wGeHJYmSeBgd71DwH/NdIfzETxjz/+G7DJOff9EcqUDvRVmNkivN9PfYLrlW1muQO38Tob1w8r9iRwvXnOBZoGmqyTYMT/0pLxfg0T+zm6AfhtnDIrgEvNLN8/FHKp/1hCmNkS4MvAFc659hHKjOd3noi6xfYrfWCE1xzP328ivBvY7Jyrircxke/ZKN8Nk/f5muge8KlwwTvL5U28sw++4j92J94fB0AI71DDNmA1MG8S6nQBXpPtdWCdf1kGfAr4lF/mFmAD3pkSK4F3TEK95vmv95r/2gPvV2y9DLjLfz/fABZO0u8xC++LPRLzWFLeL7ww2gf04P0X9gm8fqVnga3+dYFfdiHws5h9P+5/1rYBH0twnbbhHTMe+IwNnB1XDjw12u98Et6vB/zPz+t4X3Jlw+vm3z/s7zeR9fIf//eBz1VM2Ul5z0b5bpi0z5emmBARCbjpeGhIRESOgIJARCTgFAQiIgGnIBARCTgFgYhIwCkIRCaReTOm/j7Z9RCJpSAQEQk4BYFIHGZ2nZmt9uee/4mZpZpZq5n9s5m9ambPmlmxX3aBma20oTUA8v3HTzCzP/mT4r1qZsf7T59jZo+at27ArxI9863IWBQEIsOY2SnA1XiTjC0A+oC/BbLx5j06G3gB+Ad/l18AX3bOnYE3cnbg8V8BdzlvUrx34I1oBW92yc/jzTk/Dzg/4T+UyCjSkl0BkSnoEuDtwCv+P+thvAm/+hmalOyXwG/MLAJEnXMv+I//HHjEn5emwjn3OIBzrhPAf77Vzp/TxrzVsOYAf0n8jyUSn4JA5HAG/Nw5d8chD5r9/bByo83PMtrhnq6Y233o71CSTIeGRA73LPAhM5sBg2vHHof39/Ihv8xHgL8455qAg2Z2of/4R4EXnDeffJWZvd9/jkwzy5rUn0JknPSfiMgwzrmNZvZVvNWoUvBmqvw00Aa8zczW4q1qd7W/yw3APf4X/Q7gY/7jHwV+YmZ3+s/x4Un8MUTGTbOPioyTmbU653KSXQ+RiaZDQyIiAacWgYhIwKlFICIScAoCEZGAUxCIiAScgkBEJOAUBCIiAff/AQAGy4Q1C5l6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import torch.utils.data as Data\n",
    "import torch.optim as optim\n",
    "from torch.nn import init\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import numpy as np\n",
    "import random\n",
    "from IPython import display\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "#下载MNIST手写数据集 :包括训练集和测试集\n",
    "train_dataset = torchvision.datasets.MNIST(root='./Datasets/MNIST', train=True,  download=True, transform=transforms.ToTensor())  \n",
    "test_dataset = torchvision.datasets.MNIST(root='./Datasets/MNIST', train=False,  download=True, transform=transforms.ToTensor())  \n",
    "\n",
    "batch_size = 32  \n",
    "train_iter = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True,  num_workers=0)  \n",
    "test_iter = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False,  num_workers=0) \n",
    "\n",
    "#超参数初始化\n",
    "num_inputs=784 #28*28\n",
    "num_hiddens=256 #num_hiddens的值即隐层单元个数，还可为16,32,64...512,1024\n",
    "num_outputs=10\n",
    "#实现FlattenLayer层\n",
    "class FlattenLayer(nn.Module):\n",
    "    def _init_(self):\n",
    "        super(FlattenLayer,self)._init_()\n",
    "    def forward(self, x):\n",
    "        return x.view(x.shape[0],-1)#不管是否有展平的需要，都要加上，在这个例子中，显然有。\n",
    "#模型定义\n",
    "net = nn.Sequential(\n",
    "        FlattenLayer(),\n",
    "        nn.Linear(num_inputs,num_hiddens),\n",
    "        #nn.ReLU(),\n",
    "        #nn.Sigmoid(),\n",
    "        nn.Tanh(),\n",
    "        nn.Linear(num_hiddens,num_outputs)\n",
    "    )\n",
    "for param in net.parameters():\n",
    "    init.normal_(param,mean=0,std=0.01)\n",
    "lr=0.01\n",
    "loss = torch.nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(),lr)\n",
    "#返回准确率以及loss\n",
    "flag=0\n",
    "def evaluate_accuracy_loss(net, data_iter):\n",
    "    acc_sum=0.0\n",
    "    loss_sum=0.0\n",
    "    n=0\n",
    "    global flag\n",
    "    for X,y in data_iter:\n",
    "        y_hat = net(X)\n",
    "        if flag==0:print (y_hat)#测试一下y_hat是否已经softmax激活\n",
    "        flag = 1\n",
    "        acc_sum += (y_hat.argmax(dim=1)==y).sum().item()\n",
    "        l = loss(y_hat,y)\n",
    "        loss_sum += l.item()*y.shape[0]#由于loss(y_hat,y)默认为求平均，因此*y.shape[0]意味着求和。\n",
    "        n+=y.shape[0]\n",
    "    return acc_sum/n,loss_sum/n\n",
    "#记录列表（list），存储训练集和测试集上经过每一轮次，loss的变化\n",
    "def train3 (net,train_iter,test_iter,loss,num_epochs,batch_size,params = None,lr=None,optimizer=None):\n",
    "    train_loss=[]\n",
    "    test_loss=[]\n",
    "    for epoch in range(num_epochs):#外循环控制循环轮次---跑完一轮，也就把数据走了一遍\n",
    "        train_l_sum=0.0#记录训练集上的损失\n",
    "        train_acc_num=0.0#记录训练集上的准确数\n",
    "        n =0.0\n",
    "        #step1在训练集上，进行小批量梯度下降更新参数\n",
    "        for X,y in train_iter:#内循环控制训练批次\n",
    "            y_hat = net(X)\n",
    "            #保证y与y_hat维度一致，否则将会发生广播\n",
    "            l = loss(y_hat,y)#这里计算出的loss是已经求和过的，l.size = torch.Size([]),即说明loss为表示*标量*的tensor`\n",
    "            #梯度清零\n",
    "            if optimizer is not None:\n",
    "                optimizer.zero_grad()\n",
    "            elif params is not None and params[0].grad is not None:\n",
    "                for param in params:\n",
    "                    param.grad.data.zero_()\n",
    "            l.backward()\n",
    "            if optimizer is None:\n",
    "                SGD(params,lr,batch_size)\n",
    "            else:\n",
    "                optimizer.step()\n",
    "            #每一个迭代周期中得到的训练集上的loss累积进来\n",
    "            train_l_sum += l.item()*y.shape[0]\n",
    "            #计算训练样本的准确率---将每个迭代周期中预测正确的样本数累积进来\n",
    "            train_acc_num += (y_hat.argmax(dim=1)==y).sum().item()#转为int类型\n",
    "            n += y.shape[0]\n",
    "        #step2 每经过一个轮次的训练， 记录训练集和测试集上的loss\n",
    "        #注意要取平均值，loss默认求了sum\n",
    "        train_loss.append(train_l_sum/n)#训练集loss\n",
    "        test_acc,test_l = evaluate_accuracy_loss(net,test_iter)\n",
    "        test_loss.append(test_l)\n",
    "        print(\"epoch %d,train_loss %.6f,test_loss %.6f,train_acc %.6f,test_acc %.6f\"%(epoch+1,train_loss[epoch],test_loss[epoch],train_acc_num/n,test_acc)) \n",
    "    return train_loss, test_loss\n",
    "lr = 0.01\n",
    "num_epochs = 20\n",
    "train_loss,test_loss = train3(net,train_iter,test_iter,loss,num_epochs,batch_size,net.parameters,lr,optimizer)\n",
    "import matplotlib.pyplot as plt\n",
    "x=np.linspace(0,len(train_loss),len(train_loss))\n",
    "plt.plot(x,train_loss,label=\"train_loss\",linewidth=1.5)\n",
    "plt.plot(x,test_loss,label=\"test_loss\",linewidth=1.5)\n",
    "plt.xlabel(\"epoch\")\n",
    "plt.ylabel(\"loss\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, loss 0.3980, train acc 0.777, test acc 0.819\n",
      "epoch 2, loss 0.0221, train acc 0.772, test acc 0.696\n",
      "epoch 3, loss 0.0095, train acc 0.691, test acc 0.710\n",
      "epoch 4, loss 0.0069, train acc 0.703, test acc 0.709\n",
      "epoch 5, loss 0.0058, train acc 0.724, test acc 0.697\n",
      "epoch 6, loss 0.0051, train acc 0.744, test acc 0.760\n",
      "epoch 7, loss 0.0047, train acc 0.758, test acc 0.765\n",
      "epoch 8, loss 0.0043, train acc 0.773, test acc 0.783\n",
      "epoch 9, loss 0.0040, train acc 0.785, test acc 0.779\n",
      "epoch 10, loss 0.0038, train acc 0.795, test acc 0.801\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de5hU1Znv8e9b1Tf6gnJpEUEEDQgYFZIWTRw1MxoFdcTEu2Mic/TheI5OzEn0RE8uc0JOzuMkGXOZGI3J4GTUhPESDWckgxE1JmOUi5IoIOEiCS0KLfeG7qar6j1/7N3du4rqpht6d1V3/z7PU0/ty1q73i60315r7b2WuTsiIiK5EoUOQEREipMShIiI5KUEISIieSlBiIhIXkoQIiKSV0mhA+gtI0eO9PHjxxc6DBGRfmXFihXvu3ttvnMDJkGMHz+e5cuXFzoMEZF+xcz+1Nk5dTGJiEheShAiIpKXEoSIiOQ1YMYgRGRgam1tpb6+nubm5kKH0q9VVFQwduxYSktLu10n1gRhZjOB7wJJ4Mfufk8n5a4EHgfOcPfl4bG7gZuANPAZd18cZ6wiUpzq6+upqalh/PjxmFmhw+mX3J3t27dTX1/PhAkTul0vti4mM0sC9wGzgKnAdWY2NU+5GuAzwKuRY1OBa4FTgJnAD8Lricgg09zczIgRI5QcjoCZMWLEiB63wuIcg5gBrHf3je5+AFgAzM5T7mvAN4Bo5LOBBe7e4u5vA+vD6/W6nfsO8PKG91mw9M9xXF5EeoGSw5E7nO8wzi6mMcDmyH49cGa0gJlNB4539383szty6r6SU3dM7geY2VxgLsC4ceN6HGBrOsOZ/3cJB9IZAC4+bTRDK7rfPyciMpDF2YLIl67aF58wswTwbeDzPa3bfsD9QXevc/e62tq8DwJ2qTSZ4KRjqtv31763t8fXEBEZqOJMEPXA8ZH9scCWyH4N8EHgRTPbBJwFLDSzum7U7TVTjq1p337r3T1xfISI9GO7du3iBz/4QY/rXXzxxezatavH9ebMmcMTTzzR43pxiDNBLAMmmtkEMysjGHRe2HbS3Xe7+0h3H+/u4wm6lC4L72JaCFxrZuVmNgGYCCyNI8jJozsSxBq1IEQkR2cJIp1Od1lv0aJFHH300XGF1SdiG4Nw95SZ3QYsJrjNdb67rzKzecByd1/YRd1VZvYYsBpIAbe6e9f/Godp8rFD27fVghApbuPveia2a2+655K8x++66y42bNjAtGnTKC0tpbq6mtGjR7Ny5UpWr17N5ZdfzubNm2lubub2229n7ty5Qazh/HCNjY3MmjWLv/iLv+Dll19mzJgx/OIXv2DIkCGHjGnJkiXccccdpFIpzjjjDO6//37Ky8u56667WLhwISUlJVx44YV861vf4vHHH+erX/0qyWSSo446ipdeeumIv5NYn4Nw90XAopxjX+mk7Mdy9r8OfD224ELRFsTa9/aSyTiJhO6YEJHAPffcw5tvvsnKlSt58cUXueSSS3jzzTfbnyeYP38+w4cPp6mpiTPOOIMrrriCESNGZF1j3bp1/OxnP+NHP/oRV199NU8++SQ33HBDl5/b3NzMnDlzWLJkCZMmTeLTn/40999/P5/+9Kd56qmneOuttzCz9m6sefPmsXjxYsaMGXNYXVv5DPqpNmqryxlRVQbAvgNp6nc2FTgiESlmM2bMyHrY7Hvf+x6nn346Z511Fps3b2bdunUH1ZkwYQLTpk0D4MMf/jCbNm065OesXbuWCRMmMGnSJABuvPFGXnrpJYYOHUpFRQU333wzP//5z6msrATg7LPPZs6cOfzoRz86ZPdXdw36qTbMjMmja/jP9dsBWPPeHsaNqCxwVCKST2fdQH2pqqqqffvFF1/kueee43e/+x2VlZV87GMfy/swWnl5eft2MpmkqenQf4i6H3TjJgAlJSUsXbqUJUuWsGDBAr7//e/z/PPP88ADD/Dqq6/yzDPPMG3aNFauXHlQS6anBn0LAuDkUdFxCA1Ui0iHmpoa9u7N/3th9+7dDBs2jMrKSt566y1eeeWVvOUOx+TJk9m0aRPr168H4OGHH+a8886jsbGR3bt3c/HFF/Od73yHlStXArBhwwbOPPNM5s2bx8iRI9m8eXNXl++WQd+CgOxxiLfe00C1iHQYMWIEZ599Nh/84AcZMmQIo0aNaj83c+ZMHnjgAU477TROPvlkzjrrrF773IqKCh566CGuuuqq9kHqW265hR07djB79myam5txd7797W8DcOedd7Ju3TrcnfPPP5/TTz/9iGOwzpox/U1dXZ0f7opyb9Tv5q+//1sAJoys4oU7PtaLkYnIkVizZg1TpkwpdBgDQr7v0sxWuHtdvvLqYgImjqqm7calTdv3sf9AqrABiYgUASUIoKI0yYSRwcCTO/xxa2OBIxKRge7WW29l2rRpWa+HHnqo0GFl0RhEaPLooWxo2AfA2vf2MO34/v0EpIgUt/vuu6/QIRySWhCh6JxMa3Qnk4iIEkSbrCk3dCeTiIgSRJvsW133dvqQiojIYKEEERpz9BBqyoMhmV37W9m6p6XAEYmIFJYSRKhtyo02a9TNJCIc/noQAN/5znfYv39/l2XGjx/P+++/f1jXj5sSRET21N8aqBaR+BNEMdNtrhGackOkyP3yLnjvjd695rGnwqx7Oj0dXQ/i4x//OMcccwyPPfYYLS0tfOITn+CrX/0q+/bt4+qrr6a+vp50Os2Xv/xltm7dypYtW/jLv/xLRo4cyQsvvHDIUO69917mz58PwM0338xnP/vZvNe+5ppr8q4J0duUICLUghCRXNH1IJ599lmeeOIJli5dirtz2WWX8dJLL9HQ0MBxxx3HM88ECxrt3r2bo446invvvZcXXniBkSNHHvJzVqxYwUMPPcSrr76Ku3PmmWdy3nnnsXHjxoOuvWPHjrxrQvS2WBOEmc0EvkuwotyP3f2enPO3ALcCaaARmOvuq81sPLAGWBsWfcXdb4kzVoCTI89CbGhopCWVprwkGffHikh3dfGXfl949tlnefbZZ5k+fToAjY2NrFu3jnPOOYc77riDL3zhC1x66aWcc845Pb72b3/7Wz7xiU+0Tyf+yU9+kt/85jfMnDnzoGunUqn2NSEuueQSLr300l79OdvENgZhZkngPmAWMBW4zsym5hT7qbuf6u7TgG8A90bObXD3aeEr9uQAUF1ewrjhwVoQqYyzYdu+vvhYEekn3J27776blStXsnLlStavX89NN93EpEmTWLFiBaeeeip333038+bNO6xr55Pv2m1rQlxxxRU8/fTTzJw580h/tLziHKSeAax3943ufgBYAMyOFnD3aEd/FVDwhw+irQiNQ4hIdD2Iiy66iPnz59PYGMzX9s4777Bt2za2bNlCZWUlN9xwA3fccQevvfbaQXUP5dxzz+Xpp59m//797Nu3j6eeeopzzjkn77U7WxOit8XZxTQGiK5YUQ+cmVvIzG4FPgeUAX8VOTXBzF4H9gBfcvff5Kk7F5gLMG7cuF4JesqxNfxq9VYgeGBORAa36HoQs2bN4vrrr+cjH/kIANXV1TzyyCOsX7+eO++8k0QiQWlpKffffz8Ac+fOZdasWYwePfqQg9Qf+tCHmDNnDjNmzACCQerp06ezePHig669d+/evGtC9LbY1oMws6uAi9z95nD/U8AMd/+7TspfH5a/0czKgWp3325mHwaeBk7JaXFkOZL1IKIWvfEu//3RIPufM3EkD990UE4TkT6k9SB6TzGtB1EPHB/ZHwts6aL8AuByAHdvcfft4fYKYAMwKaY4s0w+NnvKDRGRwSrOLqZlwEQzmwC8A1wLXB8tYGYT3X1duHsJsC48XgvscPe0mZ0ITAQ2xhhruxNGVFFRmqC5NUPD3ha2N7Yworr80BVFRLpw5pln0tKSPYXPww8/zKmnnlqgiA4ttgTh7ikzuw1YTHCb63x3X2Vm84Dl7r4QuM3MLgBagZ3AjWH1c4F5ZpYiuAX2FnffEVesUcmEcfKoGn5fvxuAte/t5aMfUIIQKSR3x8wKHcYRefXVVwv6+YcznBDrcxDuvghYlHPsK5Ht2zup9yTwZJyxdWXysUPbE8Sa9/by0Q8c+iEXEYlHRUUF27dvZ8SIEf0+SRSKu7N9+3YqKip6VE9PUueRNeXGu7rVVaSQxo4dS319PQ0NDYUOpV+rqKhg7NixPaqjBJFH9uJBGqgWKaTS0lImTJhQ6DAGJc3mmkf0TqY/bt1LKp0pYDQiIoWhBJHHsKoyjh0a9NW1pDJs2t5/p+sVETlcShCd0NTfIjLYKUF0QlN/i8hgpwTRiSlqQYjIIKcE0YloC2KNWhAiMggpQXTixNoqSpPBQznv7GpiT3NrgSMSEelbShCdKE0mOKm2un1/rZ6HEJFBRgmiC1NGRweqNQ4hIoOLEkQXog/MrVELQkQGGSWILkxWC0JEBjEliC5MyZpyo5FMpuBLZouI9BkliC7U1pQzvKoMgMaWFO/saipwRCIifUcJogtmlj0OoW4mERlEYk0QZjbTzNaa2XozuyvP+VvM7A0zW2lmvzWzqZFzd4f11prZRXHG2RVN/S0ig1VsCcLMksB9wCxgKnBdNAGEfurup7r7NOAbwL1h3akEa1ifAswEfhBer89p0j4RGazibEHMANa7+0Z3PwAsAGZHC7h79DduFdA2CjwbWODuLe7+NrA+vF6fm6JJ+0RkkIozQYwBNkf268NjWczsVjPbQNCC+EwP6841s+Vmtjyu5QgnjqomES6D+/b2fTQdSMfyOSIixSbOBJFvdfGD7hN19/vc/STgC8CXelj3QXevc/e62traIwq2MxWlSSaMrAo/L1hhTkRkMIgzQdQDx0f2xwJbuii/ALj8MOvGKuuBOY1DiMggEWeCWAZMNLMJZlZGMOi8MFrAzCZGdi8B1oXbC4FrzazczCYAE4GlMcbapSlZt7qqBSEig0NJXBd295SZ3QYsBpLAfHdfZWbzgOXuvhC4zcwuAFqBncCNYd1VZvYYsBpIAbe6e8E6/7NvdVULQkQGh9gSBIC7LwIW5Rz7SmT79i7qfh34enzRdV/2ra57cXfM8g2TiIgMHHqSuhvGHD2EmvIgl+7a38rWPS0FjkhEJH5KEN1gZpycNfW3uplEZOBTguimrG4mDVSLyCCgBNFNGqgWkcFGCaKbpkRaEFqfWkQGAyWIbpo0qiNBrN/WyIFUpoDRiIjETwmim2oqSjl++BAAUhlnQ0NjgSMSEYmXEkQPaBxCRAYTJYgeiE65oTuZRGSgU4LogeikfWs0UC0iA5wSRA9MzmpBqItJRAY2JYgeOGFEFRWlwVe2bW8L2xs15YaIDFxKED2QTBgnj9LzECIyOChB9FD0TiaNQ4jIQKYE0UPZczJpHEJEBi4liB7KfhZCLQgRGbhiTRBmNtPM1prZejO7K8/5z5nZajP7g5ktMbMTIufSZrYyfC3MrVso0TuZ/rh1L6m0ptwQkYEptgRhZkngPmAWMBW4zsym5hR7Hahz99OAJ4BvRM41ufu08HVZXHH21LCqMkYNLQegJZVh0/b9BY5IRCQecbYgZgDr3X2jux8AFgCzowXc/QV3b/sN+wowNsZ4eo2m3BCRwSDOBDEG2BzZrw+PdeYm4JeR/QozW25mr5jZ5fkqmNncsMzyhoaGI4+4myZr6m8RGQRKYry25TnmeQua3QDUAedFDo9z9y1mdiLwvJm94e4bsi7m/iDwIEBdXV3ea8dhSvRWV83JJCIDVJwtiHrg+Mj+WGBLbiEzuwD4InCZu7c/muzuW8L3jcCLwPQYY+2RrFtd1cUkIgNUnAliGTDRzCaYWRlwLZB1N5KZTQd+SJActkWODzOz8nB7JHA2sDrGWHvkxJHVlCaDBlL9zib2NLcWOCIRkd4XW4Jw9xRwG7AYWAM85u6rzGyembXdlfRNoBp4POd21inAcjP7PfACcI+7F02CKCtJcFJtdfv+HzUOISIDUJxjELj7ImBRzrGvRLYv6KTey8CpccZ2pKaMHtr+oNya9/ZSN354gSMSEeldepL6MGnqbxEZ6JQgDlN08SBNuSEiA5ESxGGKLj+69r29ZDJ9dpetiEifUII4TLU15QyvKgOgsSXFO7uaChyRiEjvUoI4TGaWNQ6xRuMQIjLAKEEcAU39LSIDmRLEEci6k0lPVIvIAKMEcQSyV5dTC0JEBhYliCMw8ZgaEuGUhG9v30fTgXRhAxIR6UVKEEdgSFmS8SOrAHAPVpgTERkolCCOUHTqb60NISIDiRLEEcq61VUD1SIygChBHKGsKTc0UC0iA0i3EoSZ3W5mQy3wz2b2mpldGHdw/UHura7umnJDRAaG7rYg/ou77wEuBGqBvwXuiS2qfmTssCFUlwezpu/c38q2vS2HqCEi0j90N0G0rS99MfCQu/+e/GtODzqackNEBqruJogVZvYsQYJYbGY1QOZQlcxsppmtNbP1ZnZXnvOfM7PVZvYHM1tiZidEzt1oZuvC143d/YEKIXuNao1DiMjA0N0V5W4CpgEb3X2/mQ0n6GbqlJklgfuAjwP1wDIzW5izdOjrQF14zf8GfAO4Jrz+3wN1gBMkqIXuvrMnP1xfyZqTSS0IERkgutuC+Aiw1t13mdkNwJeA3YeoMwNY7+4b3f0AsACYHS3g7i+4+/5w9xVgbLh9EfArd98RJoVfATO7GWufm6IWhIgMQN1NEPcD+83sdOB/An8C/vUQdcYAmyP79eGxztwE/PIw6xbUpFEdCWL9tkYOpA7Z+yYiUvS6myBSHty/ORv4rrt/F6g5RJ18g9h57wENWyV1wDd7UtfM5prZcjNb3tDQcIhw4lNTUcrxw4cAkMo4GxoaCxaLiEhv6W6C2GtmdwOfAp4JxxdKD1GnHjg+sj8W2JJbyMwuAL4IXObuLT2p6+4Punudu9fV1tZ280eJR/baEBqHEJH+r7sJ4hqgheB5iPcIunu+2XUVlgETzWyCmZUB1wILowXMbDrwQ4LksC1yajFwoZkNM7NhBM9fLO5mrAWR9cCcnqgWkQGgWwkiTAqPAkeZ2aVAs7t3OQbh7ingNoJf7GuAx9x9lZnNM7PLwmLfBKqBx81spZktDOvuAL5GkGSWAfPCY0Ur2oJYo4FqERkAunWbq5ldTfDL/EWC8YF/MrM73f2Jruq5+yJgUc6xr0S2L+ii7nxgfnfiKwbZiwepi0lE+r/uPgfxReCMtm4gM6sFngO6TBCDyfgRVZSXJGhJZdi2t4Ud+w4wvKqs0GGJiBy27o5BJHLGCLb3oO6gkEwYJ2uNahEZQLr7S/4/zGyxmc0xsznAM+R0HYkGqkVkYOlWF5O732lmVwBnE4xBPOjuT8UaWT+kW11FZCDp7hgE7v4k8GSMsfR7mrRPRAaSLhOEme0l/9PPBri7D81zbtCanLM+dTrjJBOaFV1E+qcuE4S7H2o6DYkYXlXGqKHlbN3TQksqw6bt+ziptrrQYYmIHBbdidTLsqf+VjeTiPRfShC9LHscQgPVItJ/KUH0sinRKTfUghCRfkwJopepBSEiA4USRC87cWQ1pcngzqX6nU3saW4tcEQiIodHCaKXlZUksu5c+qOehxCRfkoJIgbRKTc09beI9FdKEDGYPDp6q6vGIUSkf1KCiEHWpH1qQYhIPxVrgjCzmWa21szWm9ldec6fa2avmVnKzK7MOZcOV5lrX2muv5gyOnvKDfd8s5WIiBS32BKEmSWB+4BZwFTgOjObmlPsz8Ac4Kd5LtHk7tPC12V5zhetY2rKGVZZCkBjS4r6nU0FjkhEpOfibEHMANa7+0Z3PwAsAGZHC7j7Jnf/A5CJMY4+Z2Y5U3+rm0lE+p84E8QYYHNkvz481l0VZrbczF4xs8t7N7T4aY1qEenvur0exGHIN891Tzrjx7n7FjM7EXjezN5w9w1ZH2A2F5gLMG7cuMOPNAZT1IIQkX4uzhZEPXB8ZH8ssKW7ld19S/i+EXgRmJ6nzIPuXufudbW1tUcWbS+LtiDWaMoNEemH4kwQy4CJZjbBzMqAa4Fu3Y1kZsPMrDzcHkmw1Onq2CKNwcRjamhbK2jT+/toOpAubEAiIj0UW4Jw9xRwG7AYWAM85u6rzGyemV0GYGZnmFk9cBXwQzNbFVafAiw3s98DLwD3uHu/ShBDypKMH1kFQMZh3TZ1M4lI/xLnGATuvghYlHPsK5HtZQRdT7n1XgZOjTO2vjDl2KFsbNgHBIsHnTb26AJHJCLSfXqSOkbZczJpHEJE+hcliBhlz8mkLiYR6V+UIGKUPSfTHk25ISL9ihJEjMYOG0J1eTDMs3N/K9v2thQ4IhGR7lOCiJGZcXJ0HEJPVItIP6IEETNN/S0i/ZUSRMwm50z9LSLSXyhBxGyKuphEpJ9SgojZpEiC2NDQyIHUgJrZXEQGMCWImA2tKGXssCEAtKadje83FjgiEZHuUYLoA1mLB+mBORHpJ5Qg+sAUTf0tIv2QEkQfUAtCRPojJYg+kLX8qFoQItJPKEH0gfEjqigvCb7qrXta2LHvQIEjEhE5NCWIPpBMZE+5oVaEiPQHShB9JGvKDY1DiEg/EGuCMLOZZrbWzNab2V15zp9rZq+ZWcrMrsw5d6OZrQtfN8YZZ1/IGqhWC0JE+oHYEoSZJYH7gFnAVOA6M5uaU+zPwBzgpzl1hwN/D5wJzAD+3syGxRVrX8geqFYLQkSKX5wtiBnAenff6O4HgAXA7GgBd9/k7n8AcuefuAj4lbvvcPedwK+AmTHGGrtoC2Lte3tJZ7R4kIgUtzgTxBhgc2S/PjzWa3XNbK6ZLTez5Q0NDYcdaF8YXlXGMTXlALSkMmzavq/AEYmIdC3OBGF5jnX3z+Zu1XX3B929zt3ramtrexRcIWiNahHpT+JMEPXA8ZH9scCWPqhbtKJTf6/VQLWIFLk4E8QyYKKZTTCzMuBaYGE36y4GLjSzYeHg9IXhsX5tctacTGpBiEhxiy1BuHsKuI3gF/sa4DF3X2Vm88zsMgAzO8PM6oGrgB+a2aqw7g7gawRJZhkwLzzWr+lWVxHpT0rivLi7LwIW5Rz7SmR7GUH3Ub6684H5ccbX106qraYkYaQyzuYdTextbqWmorTQYYmI5KUnqftQWUmCDxxT3b7/x63qZhKR4qUE0ccmZ61RrQQhIsVLCaKPZd3qqnEIESliShB9TJP2iUh/oQTRx6ZktSD24q4pN0SkOClB9LFjasoZVhncudTYkqJ+Z1OBIxIRyU8Joo+ZWc7zEOpmEpHipARRAFlTf7+rgWoRKU5KEAUwRS0IEekHlCAKILo+9Rrd6ioiRUoJogAmjarBwgnNN72/j6YD6cIGJCKShxJEAQwpSzJhRBUAGYd129TNJCLFRwmiQLRGtYgUOyWIAsm61VVPVItIEVKCKJCsKTc0UC0iRUgJokCiU26seXePptwQkaITa4Iws5lmttbM1pvZXXnOl5vZv4XnXzWz8eHx8WbWZGYrw9cDccZZCGOOHkJ1ebBe0879rTTsbSlwRCIi2WJLEGaWBO4DZgFTgevMbGpOsZuAne7+AeDbwD9Ezm1w92nh65a44iyURMJynofQOISIFJc4WxAzgPXuvtHdDwALgNk5ZWYDPwm3nwDON2t7QmDgy576W+MQIlJc4kwQY4DNkf368FjeMu6eAnYDI8JzE8zsdTP7tZmdk+8DzGyumS03s+UNDQ29G30fmDxaU26ISPGKM0HkawnkjsR2VuZdYJy7Twc+B/zUzIYeVND9QXevc/e62traIw64r03JWn5ULQgRKS5xJoh64PjI/lhgS2dlzKwEOArY4e4t7r4dwN1XABuASTHGWhCTIgliQ0MjB1KZAkYjIpItzgSxDJhoZhPMrAy4FliYU2YhcGO4fSXwvLu7mdWGg9yY2YnARGBjjLEWxNCKUsYOGwJAa9rZ+H5jgSMSEekQW4IIxxRuAxYDa4DH3H2Vmc0zs8vCYv8MjDCz9QRdSW23wp4L/MHMfk8weH2Lu++IK1b+9DKkW2O7fFf0RLWIFKuSOC/u7ouARTnHvhLZbgauylPvSeDJOGNrt+vP8NAsqBwJp10D0/8GRp3SJx8NMGV0Dc+t2QoEU39fftA4vohIYehJ6prj4PrH4ISPwtIH4f6Pwg/PhVcfhP3xNVranJx1q6taECJSPGJtQfQLyRKYdFHw2rcd3nwCXn8EfnknPPtFOHkWTLsBTvqroGwvy16fWncyiUjxUIKIqhoBZ/7X4PXeG/D6o/DGY7D6F1B9LJx+LUz7G6jtvRuqxo+opLwkQUsqw9Y9Lezcd4BhVWW9dn0RkcOlLqbOHHsqzLoHPvcWXPMIHDcdXv4nuO8M+PEFsPwhaN59xB9TkkwwaZTWhhCR4qMEcSglZTDlr+H6BfD5t+DC/wMtjfDvn4VvTYInb4YNL0Dm8J9h0NTfIlKM1MXUE9XHwEf/Dj5yG2x5DVb+FN54PHgNHQvTroNp18PwE3t02awpNzRQLSJFQi2Iw2EGYz4Ml/wjfP6PcOV8OGYy/OYf4XvT4aGLg/GLlu49+BadcuM36xr4l/98m6Vv72BPc2GezRARAbCBslBNXV2dL1++vLBB7NkCv/9ZkBx2bIDSKjjl8mBg+4SPBokljx37DvChr/0q77lxwyuZOnooU48b2v4++qgKBtGktyISIzNb4e51ec8pQcTAHTYvhZWPwJtPwYG9MGx8kChOvw6OPv6gKjf/ZBnPrdnWrcsfXVkaJIu2xHHcUE6qraY0qQahiPSMEkQhHdgHa/4frHwU3n4JMDjxvCBZTL4UyioByGScpZt2sGrLHlZt2c3qLXtYv62RVKZ7/z5lyQSTjq2OJI6jmDK6hpqK0hh/OBHp75QgisXOPwVdUCsfDab4KB8KH/xk8CDe2LqDuqBaUmnWbW1k9bt7WL1lD6vf3cOaLXvY25Lq9keeMKLyoNbGsUPVRSUiASWIYpPJwJ/+M0gUq38Brfth5CQ49WoYdgJUHA1Djs5+LwkennN36nc2sSpMGKu37GHNu3t4Z1dTtz9+WGVp1pjG1NFHcWJtlbqoRAYhJYhi1rIXVj0dJIs//67zcqWV+RPHkKOh4sCmS90AAAqESURBVCj2J6t5p7mcjY2lrN2d5M3txhvbjR2ZSlo49JPZZSUJTh5Vw9TRQ5k4qpqaihIqSpNUlCYZUppkSFnwXhHZHlKapLwkQSKh1ohIf6UE0V/s3wH7t0PTLmje1fEe3W7aFTzBHd0/0PWzE6lEOU3JGnZ5JQ2tQ9iZqWQ3Vez2KvaE77u9qv1YI0NoJUmaZPDuSVIkSBN9D15pEh1JJEwgFXkTSiLYL8sum3tsSFnH9cpLk5QkjGTCIu+J9n0lJpEj11WC0INyxaRyePDqqXQqSBp5k8kuSpp2UdO8i5qmXYxt3kVr4w5S+zeRaN5NWbqRxEErwfZMyhOk00la00nSzYkwcYTJxJPtySZvkvEgyaRI0kSSveF2iiQZEjiGO+E2wX7bywASwXiKJcAMs459M8s6lzCDRDLruJmRsESwnQjrJzqOWyIBlsQtiSeSeKIEEiW4JSFREpxLluKWxBIl7edJJLFkCW4lWKIkmOgxUQqJEiwZ1LVECZYsCY+VkEiWQTJJwpIkkwkSFiTBhEEyjNmM4LgFQ1YW/lwJA6PjfHa5nP22comO/bzlOqsXOW50xCEDjxLEQJAsCSYarBpxyKIGlIUvADJpaNnDnp0NvF3/Du+8+y67d+0g1dpKOhW+0ikyqdbglU6RSbeSyaQg1Qoe/CovId3+3vZKkqHE0iTbj2Xat5NkKCVNqaUY0kXdIOYgJSTaUwMYmfA9ctzDV+7xrBckwroJK97Wc2skcba9Z9qXcG/7aQLeyX4bd2tPrgCZrsoe4tptn912Dc/6l4h+y0ECb/tXCD43SNTefp3wTxOz4Fzbceuo4xb8i7WVcxLtnxUUsY7oLest2MrJW9HdIKkZnRW38PMPrpPNrSOm9u32n6Fju+Nc+HPkfc++Rvt7+MeQW6RcWAcSNJYOZ+QZV3DRKcceFN+RiDVBmNlM4LtAEvixu9+Tc74c+Ffgw8B24Bp33xSeuxu4CUgDn3H3xXHGOmglkjBkGEOHDOP04yZxeg+rZzJOcypNc2uGptY0TQfSNLem27ebWoP9fe37mfZj0fPZdTI0t6ZpaU2TdiedcVIZJ5V2UpkM6YzTmu6tX+4HJ47or7oEToLMQUkwaWlKo/tZCTBD0tIHJ7yspNlRrzS3vmVyrhe8Ir96of3XbPuvy3A/jN069slzPrtuxz55z+dcOyyXyPmecn/1tyXrhAXzlCVIk/DcNJIn0eccz71+IpIqOxNNaYdbpjttIrPs+BKRlJkI271tbd9EF+WSR/jHyuuZD7B41Pn9J0GEa0rfB3wcqAeWmdlCd18dKXYTsNPdP2Bm1wL/AFxjZlMJ1rA+BTgOeM7MJrl7Oq545fAkEkZlWQmVBZihPBMmjiCBZNoTSUdCyXTspzsvl85k2s+35tn3MEllPPjMjDtpd9wJj3t4HNKes99W37PrpzLQGh4PykSu5R112/bbyriDE14rLId37HvuOx37nlMua5+Dz3daL/z8tuPSW6J/lESTbyYnsRxcLk2Ca2Lo5YuzBTEDWO/uGwHMbAEwG4gmiNnA/w63nwC+b0EbbjawwN1bgLfDNatnAF3c5iODTSJhlLUPVCcLGstg5u2JIzu5wMHJyjMHJxinI/m1J7VMJ/Xb97PrtW0H8UTORWMMzxE515YAPVKP6LmsssHVwkvkfH7+z2gr3+m18sbecS0Oqtd2zcjPF5Y95bijeulftEOcCWIMsDmyXw+c2VkZd0+Z2W5gRHj8lZy6By3WbGZzgbkA48aN67XARaT72gazAZLd6piR/iLOJ6Py/ZeS2yDtrEx36uLuD7p7nbvX1dbWHkaIIiLSmTgTRD0QnZVuLLClszJmVgIcBezoZl0REYlRnAliGTDRzCaYWRnBoPPCnDILgRvD7SuB5z3onFsIXGtm5WY2AZgILI0xVhERyRHbGEQ4pnAbsJhgBHG+u68ys3nAcndfCPwz8HA4CL2DIIkQlnuMYEA7BdyqO5hERPqWptoQERnEuppqQ9N3iohIXkoQIiKS14DpYjKzBuBPhY7jCI0E3i90EEVE30c2fR8d9F1kO5Lv4wR3z/ucwIBJEAOBmS3vrC9wMNL3kU3fRwd9F9ni+j7UxSQiInkpQYiISF5KEMXlwUIHUGT0fWTT99FB30W2WL4PjUGIiEheakGIiEheShAiIpKXEkQRMLPjzewFM1tjZqvM7PZCx1RoZpY0s9fN7N8LHUuhmdnRZvaEmb0V/jfykULHVEhm9j/C/0/eNLOfmVlFoWPqS2Y238y2mdmbkWPDzexXZrYufB/WG5+lBFEcUsDn3X0KcBZwa7js6mB2O7Cm0EEUie8C/+Huk4HTGcTfi5mNAT4D1Ln7BwkmAr22sFH1uX8BZuYcuwtY4u4TgSXh/hFTgigC7v6uu78Wbu8l+AVw0Ap6g4WZjQUuAX5c6FgKzcyGAucSzHyMux9w912FjargSoAh4RoylQyytWLc/SWC2a+jZgM/Cbd/AlzeG5+lBFFkzGw8MB14tbCRFNR3gP8JZAodSBE4EWgAHgq73H5sZlWFDqpQ3P0d4FvAn4F3gd3u/mxhoyoKo9z9XQj+4ASO6Y2LKkEUETOrBp4EPuvuewodTyGY2aXANndfUehYikQJ8CHgfnefDuyjl7oP+qOwb302MAE4DqgysxsKG9XApQRRJMyslCA5POruPy90PAV0NnCZmW0CFgB/ZWaPFDakgqoH6t29rUX5BEHCGKwuAN529wZ3bwV+Dny0wDEVg61mNhogfN/WGxdVgigCZmYEfcxr3P3eQsdTSO5+t7uPdffxBIOPz7v7oP0L0d3fAzab2cnhofMJVlocrP4MnGVmleH/N+cziAftI6LLN98I/KI3LhrbkqPSI2cDnwLeMLOV4bH/5e6LChiTFI+/Ax4N13bfCPxtgeMpGHd/1cyeAF4juPvvdQbZtBtm9jPgY8BIM6sH/h64B3jMzG4iSKJX9cpnaaoNERHJR11MIiKSlxKEiIjkpQQhIiJ5KUGIiEheShAiIpKXEoRIETCzj2nmWik2ShAiIpKXEoRID5jZDWa21MxWmtkPw3UrGs3sH83sNTNbYma1YdlpZvaKmf3BzJ5qm6PfzD5gZs+Z2e/DOieFl6+OrPvwaPiksEjBKEGIdJOZTQGuAc5292lAGvgboAp4zd0/BPya4MlWgH8FvuDupwFvRI4/Ctzn7qcTzCP0bnh8OvBZYCrBLK5nx/5DiXRBU22IdN/5wIeBZeEf90MIJkXLAP8WlnkE+LmZHQUc7e6/Do//BHjczGqAMe7+FIC7NwOE11vq7vXh/kpgPPDb+H8skfyUIES6z4CfuPvdWQfNvpxTrqv5a7rqNmqJbKfR/59SYOpiEum+JcCVZnYMtK8DfALB/0dXhmWuB37r7ruBnWZ2Tnj8U8Cvw3U+6s3s8vAa5WZW2ac/hUg36S8UkW5y99Vm9iXgWTNLAK3ArQSL+JxiZiuA3QTjFBBMu/xAmACis7B+Cvihmc0Lr9ErM2+K9DbN5ipyhMys0d2rCx2HSG9TF5OIiOSlFoSIiOSlFoSIiOSlBCEiInkpQYiISF5KECIikpcShIiI5PX/AdPhBWWBZfBpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "from torch.nn import init\n",
    "import torch.utils.data\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "mnist_train = torchvision.datasets.MNIST(root='./Datasets/MNIST', train=True, download=True, transform=transforms.ToTensor())\n",
    "mnist_test = torchvision.datasets.MNIST(root='./Datasets/MNIST', train=False, download=True, transform=transforms.ToTensor())\n",
    "\n",
    "batch_size = 200\n",
    "train_iter = torch.utils.data.DataLoader(mnist_train, batch_size=batch_size, shuffle=True, num_workers=0)\n",
    "test_iter = torch.utils.data.DataLoader(mnist_test, batch_size=batch_size, shuffle=False,num_workers=0)\n",
    "\n",
    "num_inputs, num_outputs, num_hiddens1, num_hiddens2 = 784, 10, 256, 256\n",
    "\n",
    "class FlattenLayer(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(FlattenLayer, self).__init__()\n",
    "    def forward(self, x):\n",
    "        return x.view(x.shape[0], -1)\n",
    "\n",
    "\n",
    "net = nn.Sequential(  # 本次设置两层隐藏层（三层神经网络）\n",
    "     FlattenLayer(),\n",
    "     nn.Linear(num_inputs, num_hiddens1),\n",
    "     nn.ReLU(),                         \n",
    "     nn.Linear(num_hiddens1, num_hiddens2),\n",
    "     nn.ReLU(),\n",
    "     nn.Linear(num_hiddens2, num_outputs)\n",
    "     )\n",
    "\n",
    "for params in net.parameters():\n",
    "    init.normal_(params, mean=0, std=1)\n",
    "\n",
    "loss = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(net.parameters(), lr=0.03)\n",
    "\n",
    "def evaluate_accuracy(test_iter, net):\n",
    "    acc_sum, n = 0, 0\n",
    "    for X, y in test_iter:\n",
    "        y_hat = net(X)\n",
    "        acc_sum += (y_hat.argmax(dim=1) == y).sum().item()\n",
    "        n += y.shape[0]\n",
    "    return acc_sum / n\n",
    "\n",
    "num_epochs = 10\n",
    "\n",
    "def train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size, optimizer):\n",
    "    train_ls, test_ls, x_epoch = [], [], []\n",
    "    for epoch in range(num_epochs):\n",
    "        train_1_sum, train_acc_sum, n = 0.0, 0.0, 0\n",
    "        train_1_test_sum, n_test = 0.0, 0\n",
    "        for X, y in train_iter:\n",
    "            y_hat = net(X)\n",
    "            l = loss(y_hat, y).sum()\n",
    "            optimizer.zero_grad()\n",
    "            l.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            train_1_sum += l.item()\n",
    "            train_acc_sum += (y_hat.argmax(dim=1) == y).sum().item()\n",
    "            n += y.shape[0]\n",
    "        train_ls.append(train_1_sum / n)\n",
    "        x_epoch.append(epoch + 1)\n",
    "        test_acc = evaluate_accuracy(test_iter, net)\n",
    "        print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f' % (epoch + 1, train_1_sum / n, train_acc_sum / n, test_acc))\n",
    "\n",
    "        for X_test, y_test in test_iter:\n",
    "            y_hat = net(X_test)\n",
    "            l = loss(y_hat, y_test).sum()\n",
    "            train_1_test_sum += l.item()\n",
    "            n_test += y_test.shape[0]\n",
    "        test_ls.append(train_1_test_sum / n_test)\n",
    "\n",
    "\n",
    "    plt.plot(x_epoch, train_ls, label=\"train_loss\", linewidth=3)\n",
    "    plt.plot(x_epoch, test_ls, label=\"test_loss\", linewidth=1.5)\n",
    "    plt.xlabel(\"epoch\")\n",
    "    plt.ylabel(\"loss\")\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}